2024-12-07 01:22:31,186 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-07 01:22:31,196 main DEBUG Took 0.008659 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-07 01:22:31,197 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-07 01:22:31,197 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-07 01:22:31,198 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-07 01:22:31,199 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,205 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-07 01:22:31,216 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,218 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,219 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,220 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,220 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,221 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,222 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,222 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,222 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,223 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,223 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,223 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,224 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,224 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,224 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,225 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,226 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,226 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 01:22:31,226 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,226 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-07 01:22:31,228 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 01:22:31,229 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-07 01:22:31,230 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-07 01:22:31,231 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-07 01:22:31,232 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-07 01:22:31,232 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-07 01:22:31,240 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-07 01:22:31,243 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-07 01:22:31,244 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-07 01:22:31,245 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-07 01:22:31,245 main DEBUG createAppenders(={Console}) 2024-12-07 01:22:31,246 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-12-07 01:22:31,246 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-07 01:22:31,247 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-12-07 01:22:31,247 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-07 01:22:31,247 main DEBUG OutputStream closed 2024-12-07 01:22:31,248 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-07 01:22:31,248 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-07 01:22:31,248 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-12-07 01:22:31,312 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-07 01:22:31,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-07 01:22:31,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-07 01:22:31,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-07 01:22:31,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-07 01:22:31,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-07 01:22:31,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-07 01:22:31,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-07 01:22:31,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-07 01:22:31,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-07 01:22:31,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-07 01:22:31,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-07 01:22:31,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-07 01:22:31,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-07 01:22:31,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-07 01:22:31,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-07 01:22:31,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-07 01:22:31,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-07 01:22:31,323 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07 01:22:31,323 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-12-07 01:22:31,324 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-07 01:22:31,324 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-12-07T01:22:31,529 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9 2024-12-07 01:22:31,532 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-07 01:22:31,533 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07T01:22:31,541 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-07T01:22:31,565 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-07T01:22:31,606 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-07T01:22:31,606 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-07T01:22:31,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-07T01:22:31,629 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b, deleteOnExit=true 2024-12-07T01:22:31,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-07T01:22:31,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/test.cache.data in system properties and HBase conf 2024-12-07T01:22:31,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/hadoop.tmp.dir in system properties and HBase conf 2024-12-07T01:22:31,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/hadoop.log.dir in system properties and HBase conf 2024-12-07T01:22:31,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-07T01:22:31,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-07T01:22:31,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-07T01:22:31,699 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-07T01:22:31,777 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T01:22:31,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-07T01:22:31,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-07T01:22:31,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-07T01:22:31,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-07T01:22:31,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-07T01:22:31,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-07T01:22:31,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-07T01:22:31,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-07T01:22:31,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-07T01:22:31,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/nfs.dump.dir in system properties and HBase conf 2024-12-07T01:22:31,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/java.io.tmpdir in system properties and HBase conf 2024-12-07T01:22:31,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-07T01:22:31,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-07T01:22:31,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-07T01:22:32,881 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-07T01:22:32,940 INFO [Time-limited test {}] log.Log(170): Logging initialized @2367ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-07T01:22:33,000 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T01:22:33,056 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T01:22:33,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T01:22:33,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T01:22:33,079 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-07T01:22:33,094 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T01:22:33,096 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/hadoop.log.dir/,AVAILABLE} 2024-12-07T01:22:33,097 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T01:22:33,269 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/java.io.tmpdir/jetty-localhost-34341-hadoop-hdfs-3_4_1-tests_jar-_-any-6955199127283930985/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-07T01:22:33,274 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:34341} 2024-12-07T01:22:33,274 INFO [Time-limited test {}] server.Server(415): Started @2700ms 2024-12-07T01:22:33,818 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T01:22:33,824 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T01:22:33,825 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T01:22:33,825 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T01:22:33,826 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-07T01:22:33,828 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/hadoop.log.dir/,AVAILABLE} 2024-12-07T01:22:33,829 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T01:22:33,922 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/java.io.tmpdir/jetty-localhost-34129-hadoop-hdfs-3_4_1-tests_jar-_-any-17129220505610413917/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:22:33,922 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:34129} 2024-12-07T01:22:33,923 INFO [Time-limited test {}] server.Server(415): Started @3349ms 2024-12-07T01:22:33,966 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T01:22:34,072 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T01:22:34,080 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T01:22:34,085 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T01:22:34,085 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T01:22:34,085 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-07T01:22:34,086 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/hadoop.log.dir/,AVAILABLE} 2024-12-07T01:22:34,087 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T01:22:34,185 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/java.io.tmpdir/jetty-localhost-39423-hadoop-hdfs-3_4_1-tests_jar-_-any-13882333562009260478/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:22:34,186 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:39423} 2024-12-07T01:22:34,186 INFO [Time-limited test {}] server.Server(415): Started @3613ms 2024-12-07T01:22:34,189 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T01:22:34,222 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T01:22:34,227 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T01:22:34,228 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T01:22:34,228 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T01:22:34,229 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-07T01:22:34,229 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/hadoop.log.dir/,AVAILABLE} 2024-12-07T01:22:34,230 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T01:22:34,323 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/java.io.tmpdir/jetty-localhost-34265-hadoop-hdfs-3_4_1-tests_jar-_-any-7915036937995084809/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:22:34,324 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:34265} 2024-12-07T01:22:34,324 INFO [Time-limited test {}] server.Server(415): Started @3751ms 2024-12-07T01:22:34,326 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T01:22:36,404 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data3/current/BP-773003372-172.17.0.3-1733534552230/current, will proceed with Du for space computation calculation, 2024-12-07T01:22:36,404 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data1/current/BP-773003372-172.17.0.3-1733534552230/current, will proceed with Du for space computation calculation, 2024-12-07T01:22:36,404 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data4/current/BP-773003372-172.17.0.3-1733534552230/current, will proceed with Du for space computation calculation, 2024-12-07T01:22:36,404 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data2/current/BP-773003372-172.17.0.3-1733534552230/current, will proceed with Du for space computation calculation, 2024-12-07T01:22:36,439 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T01:22:36,439 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T01:22:36,471 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data5/current/BP-773003372-172.17.0.3-1733534552230/current, will proceed with Du for space computation calculation, 2024-12-07T01:22:36,472 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data6/current/BP-773003372-172.17.0.3-1733534552230/current, will proceed with Du for space computation calculation, 2024-12-07T01:22:36,486 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3ebb628efff2ef0f with lease ID 0x722608125fd6f68d: Processing first storage report for DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63 from datanode DatanodeRegistration(127.0.0.1:46343, datanodeUuid=a3c8590f-98f1-416a-9096-404b5ea22c8d, infoPort=34991, infoSecurePort=0, ipcPort=43609, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230) 2024-12-07T01:22:36,488 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3ebb628efff2ef0f with lease ID 0x722608125fd6f68d: from storage DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63 node DatanodeRegistration(127.0.0.1:46343, datanodeUuid=a3c8590f-98f1-416a-9096-404b5ea22c8d, infoPort=34991, infoSecurePort=0, ipcPort=43609, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-07T01:22:36,488 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a1152e12d28089c with lease ID 0x722608125fd6f68c: Processing first storage report for DS-69389200-bc4b-4ba0-81f8-76b097d98bbd from datanode DatanodeRegistration(127.0.0.1:33463, datanodeUuid=ae426d4e-915d-48c1-af82-321a9dd4dd4c, infoPort=44753, infoSecurePort=0, ipcPort=33055, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230) 2024-12-07T01:22:36,488 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a1152e12d28089c with lease ID 0x722608125fd6f68c: from storage DS-69389200-bc4b-4ba0-81f8-76b097d98bbd node DatanodeRegistration(127.0.0.1:33463, datanodeUuid=ae426d4e-915d-48c1-af82-321a9dd4dd4c, infoPort=44753, infoSecurePort=0, ipcPort=33055, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T01:22:36,489 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3ebb628efff2ef0f with lease ID 0x722608125fd6f68d: Processing first storage report for DS-c4633d13-5abf-49cf-afea-65dc7260e68b from datanode DatanodeRegistration(127.0.0.1:46343, datanodeUuid=a3c8590f-98f1-416a-9096-404b5ea22c8d, infoPort=34991, infoSecurePort=0, ipcPort=43609, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230) 2024-12-07T01:22:36,489 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3ebb628efff2ef0f with lease ID 0x722608125fd6f68d: from storage DS-c4633d13-5abf-49cf-afea-65dc7260e68b node DatanodeRegistration(127.0.0.1:46343, datanodeUuid=a3c8590f-98f1-416a-9096-404b5ea22c8d, infoPort=34991, infoSecurePort=0, ipcPort=43609, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T01:22:36,489 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a1152e12d28089c with lease ID 0x722608125fd6f68c: Processing first storage report for DS-e0e2c8e3-dd39-4abc-adbf-b0e7f0a4d9bb from datanode DatanodeRegistration(127.0.0.1:33463, datanodeUuid=ae426d4e-915d-48c1-af82-321a9dd4dd4c, infoPort=44753, infoSecurePort=0, ipcPort=33055, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230) 2024-12-07T01:22:36,489 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a1152e12d28089c with lease ID 0x722608125fd6f68c: from storage DS-e0e2c8e3-dd39-4abc-adbf-b0e7f0a4d9bb node DatanodeRegistration(127.0.0.1:33463, datanodeUuid=ae426d4e-915d-48c1-af82-321a9dd4dd4c, infoPort=44753, infoSecurePort=0, ipcPort=33055, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T01:22:36,497 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T01:22:36,501 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa8a031ef84a4b8f2 with lease ID 0x722608125fd6f68e: Processing first storage report for DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7 from datanode DatanodeRegistration(127.0.0.1:45293, datanodeUuid=165fafa2-1111-4fc3-aa94-022ea53b0d11, infoPort=33627, infoSecurePort=0, ipcPort=34317, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230) 2024-12-07T01:22:36,501 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa8a031ef84a4b8f2 with lease ID 0x722608125fd6f68e: from storage DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7 node DatanodeRegistration(127.0.0.1:45293, datanodeUuid=165fafa2-1111-4fc3-aa94-022ea53b0d11, infoPort=33627, infoSecurePort=0, ipcPort=34317, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T01:22:36,502 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa8a031ef84a4b8f2 with lease ID 0x722608125fd6f68e: Processing first storage report for DS-08eedccc-f815-413d-a9e3-275093dfd52a from datanode DatanodeRegistration(127.0.0.1:45293, datanodeUuid=165fafa2-1111-4fc3-aa94-022ea53b0d11, infoPort=33627, infoSecurePort=0, ipcPort=34317, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230) 2024-12-07T01:22:36,502 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa8a031ef84a4b8f2 with lease ID 0x722608125fd6f68e: from storage DS-08eedccc-f815-413d-a9e3-275093dfd52a node DatanodeRegistration(127.0.0.1:45293, datanodeUuid=165fafa2-1111-4fc3-aa94-022ea53b0d11, infoPort=33627, infoSecurePort=0, ipcPort=34317, storageInfo=lv=-57;cid=testClusterID;nsid=1856427344;c=1733534552230), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-07T01:22:36,541 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9 2024-12-07T01:22:36,631 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/zookeeper_0, clientPort=61145, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T01:22:36,651 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61145 2024-12-07T01:22:36,666 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:36,668 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:36,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741825_1001 (size=7) 2024-12-07T01:22:36,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741825_1001 (size=7) 2024-12-07T01:22:36,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741825_1001 (size=7) 2024-12-07T01:22:37,264 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe with version=8 2024-12-07T01:22:37,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/hbase-staging 2024-12-07T01:22:37,521 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ec1863dc21e5:0 server-side Connection retries=45 2024-12-07T01:22:37,530 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:37,530 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:37,535 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T01:22:37,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:37,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T01:22:37,678 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T01:22:37,734 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-07T01:22:37,743 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-07T01:22:37,747 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T01:22:37,772 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 66465 (auto-detected) 2024-12-07T01:22:37,773 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-07T01:22:37,791 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45579 2024-12-07T01:22:37,809 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45579 connecting to ZooKeeper ensemble=127.0.0.1:61145 2024-12-07T01:22:37,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:455790x0, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T01:22:37,947 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45579-0x101ad621d6e0000 connected 2024-12-07T01:22:38,043 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:38,047 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:38,058 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:22:38,061 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe, hbase.cluster.distributed=false 2024-12-07T01:22:38,080 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T01:22:38,085 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45579 2024-12-07T01:22:38,085 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45579 2024-12-07T01:22:38,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45579 2024-12-07T01:22:38,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45579 2024-12-07T01:22:38,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45579 2024-12-07T01:22:38,176 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec1863dc21e5:0 server-side Connection retries=45 2024-12-07T01:22:38,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:38,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:38,178 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T01:22:38,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:38,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T01:22:38,181 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T01:22:38,183 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T01:22:38,184 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35781 2024-12-07T01:22:38,185 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35781 connecting to ZooKeeper ensemble=127.0.0.1:61145 2024-12-07T01:22:38,186 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:38,190 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:38,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:357810x0, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T01:22:38,200 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:22:38,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35781-0x101ad621d6e0001 connected 2024-12-07T01:22:38,204 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T01:22:38,211 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T01:22:38,214 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T01:22:38,220 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T01:22:38,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35781 2024-12-07T01:22:38,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35781 2024-12-07T01:22:38,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35781 2024-12-07T01:22:38,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35781 2024-12-07T01:22:38,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35781 2024-12-07T01:22:38,237 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec1863dc21e5:0 server-side Connection retries=45 2024-12-07T01:22:38,238 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:38,238 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:38,239 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T01:22:38,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:38,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T01:22:38,239 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T01:22:38,239 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T01:22:38,240 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35853 2024-12-07T01:22:38,242 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35853 connecting to ZooKeeper ensemble=127.0.0.1:61145 2024-12-07T01:22:38,243 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:38,246 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:38,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358530x0, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T01:22:38,264 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:358530x0, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:22:38,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35853-0x101ad621d6e0002 connected 2024-12-07T01:22:38,265 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T01:22:38,266 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T01:22:38,267 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T01:22:38,269 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T01:22:38,270 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35853 2024-12-07T01:22:38,270 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35853 2024-12-07T01:22:38,271 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35853 2024-12-07T01:22:38,271 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35853 2024-12-07T01:22:38,271 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35853 2024-12-07T01:22:38,286 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec1863dc21e5:0 server-side Connection retries=45 2024-12-07T01:22:38,286 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:38,287 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:38,287 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T01:22:38,287 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T01:22:38,287 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T01:22:38,287 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T01:22:38,287 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T01:22:38,288 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38575 2024-12-07T01:22:38,290 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38575 connecting to ZooKeeper ensemble=127.0.0.1:61145 2024-12-07T01:22:38,291 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:38,293 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:38,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385750x0, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T01:22:38,306 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:385750x0, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:22:38,306 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38575-0x101ad621d6e0003 connected 2024-12-07T01:22:38,307 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T01:22:38,307 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T01:22:38,308 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T01:22:38,310 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T01:22:38,311 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38575 2024-12-07T01:22:38,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38575 2024-12-07T01:22:38,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38575 2024-12-07T01:22:38,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38575 2024-12-07T01:22:38,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38575 2024-12-07T01:22:38,327 DEBUG [M:0;ec1863dc21e5:45579 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec1863dc21e5:45579 2024-12-07T01:22:38,328 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ec1863dc21e5,45579,1733534557376 2024-12-07T01:22:38,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:22:38,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:22:38,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:22:38,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:22:38,348 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec1863dc21e5,45579,1733534557376 2024-12-07T01:22:38,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T01:22:38,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T01:22:38,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:38,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:38,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T01:22:38,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:38,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:38,380 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T01:22:38,381 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec1863dc21e5,45579,1733534557376 from backup master directory 2024-12-07T01:22:38,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:22:38,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec1863dc21e5,45579,1733534557376 2024-12-07T01:22:38,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:22:38,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:22:38,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T01:22:38,390 WARN [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T01:22:38,390 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec1863dc21e5,45579,1733534557376 2024-12-07T01:22:38,392 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-07T01:22:38,393 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-07T01:22:38,448 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/hbase.id] with ID: 39d45156-d657-4acd-921b-b0e3165a848f 2024-12-07T01:22:38,448 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/.tmp/hbase.id 2024-12-07T01:22:38,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741826_1002 (size=42) 2024-12-07T01:22:38,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741826_1002 (size=42) 2024-12-07T01:22:38,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741826_1002 (size=42) 2024-12-07T01:22:38,463 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/.tmp/hbase.id]:[hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/hbase.id] 2024-12-07T01:22:38,505 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T01:22:38,509 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T01:22:38,528 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-07T01:22:38,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:38,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:38,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:38,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:38,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741827_1003 (size=196) 2024-12-07T01:22:38,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741827_1003 (size=196) 2024-12-07T01:22:38,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741827_1003 (size=196) 2024-12-07T01:22:38,569 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T01:22:38,571 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T01:22:38,576 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:38,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741828_1004 (size=1189) 2024-12-07T01:22:38,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741828_1004 (size=1189) 2024-12-07T01:22:38,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741828_1004 (size=1189) 2024-12-07T01:22:38,619 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store 2024-12-07T01:22:38,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741829_1005 (size=34) 2024-12-07T01:22:38,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741829_1005 (size=34) 2024-12-07T01:22:38,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741829_1005 (size=34) 2024-12-07T01:22:38,642 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-07T01:22:38,645 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:38,647 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T01:22:38,647 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:22:38,647 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:22:38,649 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T01:22:38,649 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:22:38,649 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:22:38,650 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733534558647Disabling compacts and flushes for region at 1733534558647Disabling writes for close at 1733534558649 (+2 ms)Writing region close event to WAL at 1733534558649Closed at 1733534558649 2024-12-07T01:22:38,652 WARN [master/ec1863dc21e5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/.initializing 2024-12-07T01:22:38,652 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/WALs/ec1863dc21e5,45579,1733534557376 2024-12-07T01:22:38,659 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:38,673 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C45579%2C1733534557376, suffix=, logDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/WALs/ec1863dc21e5,45579,1733534557376, archiveDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/oldWALs, maxLogs=10 2024-12-07T01:22:38,696 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/WALs/ec1863dc21e5,45579,1733534557376/ec1863dc21e5%2C45579%2C1733534557376.1733534558677, exclude list is [], retry=0 2024-12-07T01:22:38,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:38,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:38,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:38,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:38,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-07T01:22:38,753 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/WALs/ec1863dc21e5,45579,1733534557376/ec1863dc21e5%2C45579%2C1733534557376.1733534558677 2024-12-07T01:22:38,754 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627)] 2024-12-07T01:22:38,754 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:38,755 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:38,758 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,758 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T01:22:38,816 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:38,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:22:38,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T01:22:38,822 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:38,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:38,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T01:22:38,826 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:38,827 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:38,827 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T01:22:38,829 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:38,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:38,831 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,835 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,836 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,841 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,842 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,845 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T01:22:38,849 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T01:22:38,853 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:38,855 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74183100, jitterRate=0.10541433095932007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T01:22:38,862 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733534558770Initializing all the Stores at 1733534558772 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534558773 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534558773Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534558774 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534558774Cleaning up temporary data from old regions at 1733534558842 (+68 ms)Region opened successfully at 1733534558862 (+20 ms) 2024-12-07T01:22:38,863 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T01:22:38,895 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2b36fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec1863dc21e5/172.17.0.3:0 2024-12-07T01:22:38,921 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T01:22:38,930 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T01:22:38,930 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T01:22:38,933 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T01:22:38,934 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-07T01:22:38,938 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-07T01:22:38,938 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T01:22:38,962 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T01:22:38,969 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T01:22:39,031 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T01:22:39,036 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T01:22:39,038 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T01:22:39,052 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T01:22:39,054 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T01:22:39,056 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T01:22:39,062 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T01:22:39,064 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T01:22:39,073 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T01:22:39,094 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T01:22:39,104 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T01:22:39,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T01:22:39,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T01:22:39,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T01:22:39,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T01:22:39,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,120 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ec1863dc21e5,45579,1733534557376, sessionid=0x101ad621d6e0000, setting cluster-up flag (Was=false) 2024-12-07T01:22:39,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,179 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T01:22:39,183 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec1863dc21e5,45579,1733534557376 2024-12-07T01:22:39,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:39,242 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T01:22:39,246 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec1863dc21e5,45579,1733534557376 2024-12-07T01:22:39,255 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T01:22:39,314 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T01:22:39,318 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(746): ClusterId : 39d45156-d657-4acd-921b-b0e3165a848f 2024-12-07T01:22:39,318 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(746): ClusterId : 39d45156-d657-4acd-921b-b0e3165a848f 2024-12-07T01:22:39,318 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(746): ClusterId : 39d45156-d657-4acd-921b-b0e3165a848f 2024-12-07T01:22:39,321 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T01:22:39,321 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T01:22:39,321 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T01:22:39,325 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T01:22:39,331 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T01:22:39,336 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec1863dc21e5,45579,1733534557376 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T01:22:39,347 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T01:22:39,347 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T01:22:39,347 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T01:22:39,347 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T01:22:39,347 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T01:22:39,347 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T01:22:39,348 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec1863dc21e5:0, corePoolSize=5, maxPoolSize=5 2024-12-07T01:22:39,348 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec1863dc21e5:0, corePoolSize=5, maxPoolSize=5 2024-12-07T01:22:39,348 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec1863dc21e5:0, corePoolSize=5, maxPoolSize=5 2024-12-07T01:22:39,348 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec1863dc21e5:0, corePoolSize=5, maxPoolSize=5 2024-12-07T01:22:39,349 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec1863dc21e5:0, corePoolSize=10, maxPoolSize=10 2024-12-07T01:22:39,349 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,349 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec1863dc21e5:0, corePoolSize=2, maxPoolSize=2 2024-12-07T01:22:39,349 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,351 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733534589351 2024-12-07T01:22:39,353 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T01:22:39,354 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T01:22:39,355 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T01:22:39,356 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T01:22:39,357 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T01:22:39,358 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T01:22:39,358 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T01:22:39,358 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T01:22:39,359 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T01:22:39,359 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T01:22:39,360 DEBUG [RS:1;ec1863dc21e5:35853 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ad57d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec1863dc21e5/172.17.0.3:0 2024-12-07T01:22:39,360 DEBUG [RS:0;ec1863dc21e5:35781 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6559f142, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec1863dc21e5/172.17.0.3:0 2024-12-07T01:22:39,360 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T01:22:39,360 DEBUG [RS:2;ec1863dc21e5:38575 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2393de11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec1863dc21e5/172.17.0.3:0 2024-12-07T01:22:39,359 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,364 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:39,365 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T01:22:39,367 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T01:22:39,368 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T01:22:39,369 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T01:22:39,374 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T01:22:39,375 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T01:22:39,377 DEBUG [RS:1;ec1863dc21e5:35853 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ec1863dc21e5:35853 2024-12-07T01:22:39,377 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.large.0-1733534559377,5,FailOnTimeoutGroup] 2024-12-07T01:22:39,378 DEBUG [RS:0;ec1863dc21e5:35781 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec1863dc21e5:35781 2024-12-07T01:22:39,378 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.small.0-1733534559378,5,FailOnTimeoutGroup] 2024-12-07T01:22:39,378 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,378 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T01:22:39,379 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,380 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,380 DEBUG [RS:2;ec1863dc21e5:38575 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ec1863dc21e5:38575 2024-12-07T01:22:39,380 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T01:22:39,380 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T01:22:39,380 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T01:22:39,380 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T01:22:39,380 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T01:22:39,380 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T01:22:39,381 DEBUG [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T01:22:39,381 DEBUG [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T01:22:39,381 DEBUG [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T01:22:39,383 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec1863dc21e5,45579,1733534557376 with port=38575, startcode=1733534558286 2024-12-07T01:22:39,383 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec1863dc21e5,45579,1733534557376 with port=35853, startcode=1733534558237 2024-12-07T01:22:39,383 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec1863dc21e5,45579,1733534557376 with port=35781, startcode=1733534558145 2024-12-07T01:22:39,396 DEBUG [RS:2;ec1863dc21e5:38575 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T01:22:39,396 DEBUG [RS:0;ec1863dc21e5:35781 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T01:22:39,396 DEBUG [RS:1;ec1863dc21e5:35853 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T01:22:39,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741831_1007 (size=1321) 2024-12-07T01:22:39,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741831_1007 (size=1321) 2024-12-07T01:22:39,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741831_1007 (size=1321) 2024-12-07T01:22:39,418 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T01:22:39,418 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe 2024-12-07T01:22:39,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741832_1008 (size=32) 2024-12-07T01:22:39,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741832_1008 (size=32) 2024-12-07T01:22:39,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741832_1008 (size=32) 2024-12-07T01:22:39,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:39,439 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37201, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T01:22:39,439 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56733, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T01:22:39,439 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55483, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T01:22:39,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T01:22:39,444 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T01:22:39,444 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:39,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:22:39,445 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45579 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec1863dc21e5,35781,1733534558145 2024-12-07T01:22:39,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T01:22:39,447 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45579 {}] master.ServerManager(517): Registering regionserver=ec1863dc21e5,35781,1733534558145 2024-12-07T01:22:39,448 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T01:22:39,448 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:39,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:22:39,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T01:22:39,452 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T01:22:39,452 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:39,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:22:39,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T01:22:39,457 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T01:22:39,457 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:39,458 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:22:39,459 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T01:22:39,460 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45579 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:39,461 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45579 {}] master.ServerManager(517): Registering regionserver=ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:39,461 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740 2024-12-07T01:22:39,462 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740 2024-12-07T01:22:39,465 DEBUG [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe 2024-12-07T01:22:39,465 DEBUG [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36673 2024-12-07T01:22:39,465 DEBUG [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T01:22:39,465 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T01:22:39,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T01:22:39,466 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45579 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:39,467 DEBUG [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe 2024-12-07T01:22:39,467 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45579 {}] master.ServerManager(517): Registering regionserver=ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:39,467 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T01:22:39,467 DEBUG [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36673 2024-12-07T01:22:39,467 DEBUG [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T01:22:39,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T01:22:39,471 DEBUG [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe 2024-12-07T01:22:39,471 DEBUG [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36673 2024-12-07T01:22:39,471 DEBUG [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T01:22:39,475 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:39,476 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61789295, jitterRate=-0.07926775515079498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T01:22:39,478 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733534559439Initializing all the Stores at 1733534559441 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534559441Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534559441Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534559441Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534559441Cleaning up temporary data from old regions at 1733534559466 (+25 ms)Region opened successfully at 1733534559478 (+12 ms) 2024-12-07T01:22:39,478 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T01:22:39,478 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T01:22:39,479 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T01:22:39,479 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T01:22:39,479 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T01:22:39,480 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T01:22:39,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733534559478Disabling compacts and flushes for region at 1733534559478Disabling writes for close at 1733534559479 (+1 ms)Writing region close event to WAL at 1733534559480 (+1 ms)Closed at 1733534559480 2024-12-07T01:22:39,483 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T01:22:39,484 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T01:22:39,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T01:22:39,497 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:22:39,501 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T01:22:39,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T01:22:39,526 DEBUG [RS:0;ec1863dc21e5:35781 {}] zookeeper.ZKUtil(111): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec1863dc21e5,35781,1733534558145 2024-12-07T01:22:39,526 DEBUG [RS:2;ec1863dc21e5:38575 {}] zookeeper.ZKUtil(111): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:39,526 WARN [RS:2;ec1863dc21e5:38575 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T01:22:39,526 WARN [RS:0;ec1863dc21e5:35781 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T01:22:39,527 INFO [RS:0;ec1863dc21e5:35781 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:39,527 INFO [RS:2;ec1863dc21e5:38575 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:39,527 DEBUG [RS:1;ec1863dc21e5:35853 {}] zookeeper.ZKUtil(111): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:39,527 WARN [RS:1;ec1863dc21e5:35853 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T01:22:39,527 DEBUG [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:39,527 DEBUG [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35781,1733534558145 2024-12-07T01:22:39,527 INFO [RS:1;ec1863dc21e5:35853 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:39,527 DEBUG [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:39,528 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec1863dc21e5,38575,1733534558286] 2024-12-07T01:22:39,529 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec1863dc21e5,35781,1733534558145] 2024-12-07T01:22:39,529 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec1863dc21e5,35853,1733534558237] 2024-12-07T01:22:39,555 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T01:22:39,555 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T01:22:39,555 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T01:22:39,567 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T01:22:39,567 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T01:22:39,568 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T01:22:39,574 INFO [RS:1;ec1863dc21e5:35853 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T01:22:39,574 INFO [RS:2;ec1863dc21e5:38575 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T01:22:39,574 INFO [RS:0;ec1863dc21e5:35781 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T01:22:39,574 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,574 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,574 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,576 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T01:22:39,576 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T01:22:39,576 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T01:22:39,582 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T01:22:39,582 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T01:22:39,582 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T01:22:39,584 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,584 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,584 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,584 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,584 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,584 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,585 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0, corePoolSize=2, maxPoolSize=2 2024-12-07T01:22:39,585 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0, corePoolSize=2, maxPoolSize=2 2024-12-07T01:22:39,585 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0, corePoolSize=2, maxPoolSize=2 2024-12-07T01:22:39,585 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec1863dc21e5:0, corePoolSize=1, maxPoolSize=1 2024-12-07T01:22:39,586 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:22:39,586 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:22:39,586 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:22:39,586 DEBUG [RS:0;ec1863dc21e5:35781 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:22:39,587 DEBUG [RS:2;ec1863dc21e5:38575 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:22:39,587 DEBUG [RS:1;ec1863dc21e5:35853 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec1863dc21e5:0, corePoolSize=3, maxPoolSize=3 2024-12-07T01:22:39,588 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,588 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,35853,1733534558237-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,35781,1733534558145-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T01:22:39,589 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,38575,1733534558286-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T01:22:39,608 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T01:22:39,608 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T01:22:39,610 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,35853,1733534558237-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,610 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,35781,1733534558145-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,610 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,610 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,610 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.Replication(171): ec1863dc21e5,35853,1733534558237 started 2024-12-07T01:22:39,610 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.Replication(171): ec1863dc21e5,35781,1733534558145 started 2024-12-07T01:22:39,612 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T01:22:39,612 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,38575,1733534558286-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,613 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,613 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.Replication(171): ec1863dc21e5,38575,1733534558286 started 2024-12-07T01:22:39,635 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,635 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,635 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:39,635 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(1482): Serving as ec1863dc21e5,35781,1733534558145, RpcServer on ec1863dc21e5/172.17.0.3:35781, sessionid=0x101ad621d6e0001 2024-12-07T01:22:39,635 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(1482): Serving as ec1863dc21e5,35853,1733534558237, RpcServer on ec1863dc21e5/172.17.0.3:35853, sessionid=0x101ad621d6e0002 2024-12-07T01:22:39,635 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(1482): Serving as ec1863dc21e5,38575,1733534558286, RpcServer on ec1863dc21e5/172.17.0.3:38575, sessionid=0x101ad621d6e0003 2024-12-07T01:22:39,636 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T01:22:39,636 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T01:22:39,636 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T01:22:39,636 DEBUG [RS:1;ec1863dc21e5:35853 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:39,636 DEBUG [RS:2;ec1863dc21e5:38575 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:39,636 DEBUG [RS:0;ec1863dc21e5:35781 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec1863dc21e5,35781,1733534558145 2024-12-07T01:22:39,637 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,38575,1733534558286' 2024-12-07T01:22:39,637 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,35853,1733534558237' 2024-12-07T01:22:39,637 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,35781,1733534558145' 2024-12-07T01:22:39,637 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T01:22:39,637 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T01:22:39,637 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T01:22:39,638 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T01:22:39,638 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T01:22:39,638 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T01:22:39,638 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T01:22:39,638 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T01:22:39,639 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T01:22:39,639 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T01:22:39,639 DEBUG [RS:1;ec1863dc21e5:35853 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:39,639 DEBUG [RS:0;ec1863dc21e5:35781 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec1863dc21e5,35781,1733534558145 2024-12-07T01:22:39,639 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,35853,1733534558237' 2024-12-07T01:22:39,639 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,35781,1733534558145' 2024-12-07T01:22:39,639 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T01:22:39,639 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T01:22:39,639 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T01:22:39,639 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T01:22:39,639 DEBUG [RS:2;ec1863dc21e5:38575 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:39,639 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec1863dc21e5,38575,1733534558286' 2024-12-07T01:22:39,639 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T01:22:39,639 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T01:22:39,639 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T01:22:39,640 DEBUG [RS:1;ec1863dc21e5:35853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T01:22:39,640 DEBUG [RS:0;ec1863dc21e5:35781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T01:22:39,640 INFO [RS:1;ec1863dc21e5:35853 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T01:22:39,640 INFO [RS:0;ec1863dc21e5:35781 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T01:22:39,640 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T01:22:39,640 INFO [RS:0;ec1863dc21e5:35781 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T01:22:39,640 INFO [RS:1;ec1863dc21e5:35853 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T01:22:39,641 DEBUG [RS:2;ec1863dc21e5:38575 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T01:22:39,641 INFO [RS:2;ec1863dc21e5:38575 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T01:22:39,641 INFO [RS:2;ec1863dc21e5:38575 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T01:22:39,652 WARN [ec1863dc21e5:45579 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T01:22:39,745 INFO [RS:2;ec1863dc21e5:38575 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:39,745 INFO [RS:1;ec1863dc21e5:35853 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:39,745 INFO [RS:0;ec1863dc21e5:35781 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:39,747 INFO [RS:1;ec1863dc21e5:35853 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C35853%2C1733534558237, suffix=, logDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35853,1733534558237, archiveDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/oldWALs, maxLogs=32 2024-12-07T01:22:39,747 INFO [RS:0;ec1863dc21e5:35781 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C35781%2C1733534558145, suffix=, logDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35781,1733534558145, archiveDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/oldWALs, maxLogs=32 2024-12-07T01:22:39,747 INFO [RS:2;ec1863dc21e5:38575 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C38575%2C1733534558286, suffix=, logDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286, archiveDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/oldWALs, maxLogs=32 2024-12-07T01:22:39,764 DEBUG [RS:2;ec1863dc21e5:38575 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286/ec1863dc21e5%2C38575%2C1733534558286.1733534559752, exclude list is [], retry=0 2024-12-07T01:22:39,764 DEBUG [RS:1;ec1863dc21e5:35853 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35853,1733534558237/ec1863dc21e5%2C35853%2C1733534558237.1733534559752, exclude list is [], retry=0 2024-12-07T01:22:39,764 DEBUG [RS:0;ec1863dc21e5:35781 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35781,1733534558145/ec1863dc21e5%2C35781%2C1733534558145.1733534559752, exclude list is [], retry=0 2024-12-07T01:22:39,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:39,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:39,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:39,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:39,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:39,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:39,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:39,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:39,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:39,775 INFO [RS:2;ec1863dc21e5:38575 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286/ec1863dc21e5%2C38575%2C1733534558286.1733534559752 2024-12-07T01:22:39,775 INFO [RS:1;ec1863dc21e5:35853 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35853,1733534558237/ec1863dc21e5%2C35853%2C1733534558237.1733534559752 2024-12-07T01:22:39,776 DEBUG [RS:2;ec1863dc21e5:38575 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627)] 2024-12-07T01:22:39,776 DEBUG [RS:1;ec1863dc21e5:35853 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:39,776 INFO [RS:0;ec1863dc21e5:35781 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35781,1733534558145/ec1863dc21e5%2C35781%2C1733534558145.1733534559752 2024-12-07T01:22:39,776 DEBUG [RS:0;ec1863dc21e5:35781 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:39,905 DEBUG [ec1863dc21e5:45579 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-07T01:22:39,914 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(204): Hosts are {ec1863dc21e5=0} racks are {/default-rack=0} 2024-12-07T01:22:39,921 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T01:22:39,921 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T01:22:39,921 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-07T01:22:39,921 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T01:22:39,921 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T01:22:39,921 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-07T01:22:39,921 INFO [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T01:22:39,921 INFO [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T01:22:39,921 INFO [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-07T01:22:39,921 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T01:22:39,927 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:39,934 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,35853,1733534558237, state=OPENING 2024-12-07T01:22:40,124 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T01:22:40,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:40,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:40,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:40,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:22:40,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:22:40,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:22:40,193 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:22:40,193 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:22:40,195 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T01:22:40,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,35853,1733534558237}] 2024-12-07T01:22:40,375 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T01:22:40,377 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41433, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T01:22:40,388 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T01:22:40,388 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:40,389 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T01:22:40,392 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec1863dc21e5%2C35853%2C1733534558237.meta, suffix=.meta, logDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35853,1733534558237, archiveDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/oldWALs, maxLogs=32 2024-12-07T01:22:40,407 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35853,1733534558237/ec1863dc21e5%2C35853%2C1733534558237.meta.1733534560393.meta, exclude list is [], retry=0 2024-12-07T01:22:40,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:40,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:40,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:40,415 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,35853,1733534558237/ec1863dc21e5%2C35853%2C1733534558237.meta.1733534560393.meta 2024-12-07T01:22:40,415 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991)] 2024-12-07T01:22:40,416 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:40,417 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T01:22:40,420 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T01:22:40,424 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T01:22:40,427 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T01:22:40,428 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:40,428 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T01:22:40,428 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T01:22:40,432 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T01:22:40,433 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T01:22:40,434 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:40,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:22:40,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T01:22:40,436 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T01:22:40,437 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:40,437 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:22:40,438 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T01:22:40,439 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T01:22:40,439 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:40,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:22:40,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T01:22:40,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T01:22:40,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:40,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T01:22:40,444 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T01:22:40,445 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740 2024-12-07T01:22:40,449 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740 2024-12-07T01:22:40,451 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T01:22:40,451 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T01:22:40,452 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T01:22:40,455 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T01:22:40,457 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74552098, jitterRate=0.11091282963752747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T01:22:40,457 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T01:22:40,459 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733534560429Writing region info on filesystem at 1733534560429Initializing all the Stores at 1733534560431 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534560431Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534560431Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534560431Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733534560431Cleaning up temporary data from old regions at 1733534560451 (+20 ms)Running coprocessor post-open hooks at 1733534560457 (+6 ms)Region opened successfully at 1733534560458 (+1 ms) 2024-12-07T01:22:40,467 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733534560365 2024-12-07T01:22:40,479 DEBUG [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T01:22:40,479 INFO [RS_OPEN_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T01:22:40,481 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:40,483 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec1863dc21e5,35853,1733534558237, state=OPEN 2024-12-07T01:22:40,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:22:40,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:22:40,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:22:40,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T01:22:40,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:22:40,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:22:40,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:22:40,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T01:22:40,495 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:40,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T01:22:40,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec1863dc21e5,35853,1733534558237 in 297 msec 2024-12-07T01:22:40,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T01:22:40,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0140 sec 2024-12-07T01:22:40,509 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T01:22:40,509 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T01:22:40,532 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T01:22:40,533 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,35853,1733534558237, seqNum=-1] 2024-12-07T01:22:40,555 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T01:22:40,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35925, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T01:22:40,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2960 sec 2024-12-07T01:22:40,579 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733534560579, completionTime=-1 2024-12-07T01:22:40,582 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-07T01:22:40,582 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T01:22:40,607 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-07T01:22:40,607 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733534620607 2024-12-07T01:22:40,607 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733534680607 2024-12-07T01:22:40,607 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-12-07T01:22:40,609 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-07T01:22:40,634 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,45579,1733534557376-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:40,634 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,45579,1733534557376-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:40,634 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,45579,1733534557376-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:40,636 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec1863dc21e5:45579, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:40,638 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:40,645 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:40,650 DEBUG [master/ec1863dc21e5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T01:22:40,678 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.288sec 2024-12-07T01:22:40,679 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T01:22:40,681 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T01:22:40,682 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T01:22:40,682 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T01:22:40,682 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T01:22:40,683 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,45579,1733534557376-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T01:22:40,683 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,45579,1733534557376-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T01:22:40,688 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T01:22:40,689 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T01:22:40,689 INFO [master/ec1863dc21e5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec1863dc21e5,45579,1733534557376-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:40,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4696d0fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T01:22:40,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ec1863dc21e5,45579,-1 for getting cluster id 2024-12-07T01:22:40,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T01:22:40,762 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '39d45156-d657-4acd-921b-b0e3165a848f' 2024-12-07T01:22:40,764 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T01:22:40,764 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "39d45156-d657-4acd-921b-b0e3165a848f" 2024-12-07T01:22:40,766 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d886a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T01:22:40,766 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ec1863dc21e5,45579,-1] 2024-12-07T01:22:40,768 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T01:22:40,770 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:22:40,771 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T01:22:40,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5830ab9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T01:22:40,775 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T01:22:40,781 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec1863dc21e5,35853,1733534558237, seqNum=-1] 2024-12-07T01:22:40,782 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T01:22:40,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T01:22:40,808 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ec1863dc21e5,45579,1733534557376 2024-12-07T01:22:40,809 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:36673/hbase 2024-12-07T01:22:40,826 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=369, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=5307 2024-12-07T01:22:40,850 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:40,854 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:40,856 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:40,861 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26466257, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/hregion-26466257, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:40,881 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26466257/hregion-26466257.1733534560863, exclude list is [], retry=0 2024-12-07T01:22:40,886 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:40,886 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:40,886 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:40,889 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26466257/hregion-26466257.1733534560863 2024-12-07T01:22:40,889 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:40,889 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 652f2506c256e0809ec156571eccb513, NAME => 'testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/hbase 2024-12-07T01:22:40,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741838_1014 (size=64) 2024-12-07T01:22:40,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741838_1014 (size=64) 2024-12-07T01:22:40,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741838_1014 (size=64) 2024-12-07T01:22:40,903 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:40,905 INFO [StoreOpener-652f2506c256e0809ec156571eccb513-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:40,907 INFO [StoreOpener-652f2506c256e0809ec156571eccb513-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 652f2506c256e0809ec156571eccb513 columnFamilyName a 2024-12-07T01:22:40,908 DEBUG [StoreOpener-652f2506c256e0809ec156571eccb513-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:40,909 INFO [StoreOpener-652f2506c256e0809ec156571eccb513-1 {}] regionserver.HStore(327): Store=652f2506c256e0809ec156571eccb513/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:40,909 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:40,910 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513 2024-12-07T01:22:40,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513 2024-12-07T01:22:40,911 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:40,911 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:40,914 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:40,918 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:40,919 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 652f2506c256e0809ec156571eccb513; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64703948, jitterRate=-0.03583604097366333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:22:40,920 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 652f2506c256e0809ec156571eccb513: Writing region info on filesystem at 1733534560903Initializing all the Stores at 1733534560905 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534560905Cleaning up temporary data from old regions at 1733534560911 (+6 ms)Region opened successfully at 1733534560920 (+9 ms) 2024-12-07T01:22:40,920 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 652f2506c256e0809ec156571eccb513, disabling compactions & flushes 2024-12-07T01:22:40,920 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513. 2024-12-07T01:22:40,920 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513. 2024-12-07T01:22:40,920 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513. after waiting 0 ms 2024-12-07T01:22:40,920 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513. 2024-12-07T01:22:40,921 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513. 2024-12-07T01:22:40,921 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 652f2506c256e0809ec156571eccb513: Waiting for close lock at 1733534560920Disabling compacts and flushes for region at 1733534560920Disabling writes for close at 1733534560920Writing region close event to WAL at 1733534560921 (+1 ms)Closed at 1733534560921 2024-12-07T01:22:40,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741837_1013 (size=93) 2024-12-07T01:22:40,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741837_1013 (size=93) 2024-12-07T01:22:40,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741837_1013 (size=93) 2024-12-07T01:22:40,934 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:22:40,934 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26466257:(num 1733534560863) 2024-12-07T01:22:40,936 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-07T01:22:40,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741839_1015 (size=369) 2024-12-07T01:22:40,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741839_1015 (size=369) 2024-12-07T01:22:40,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741839_1015 (size=369) 2024-12-07T01:22:40,958 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-07T01:22:40,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741840_1016 (size=231) 2024-12-07T01:22:40,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741840_1016 (size=231) 2024-12-07T01:22:40,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741840_1016 (size=231) 2024-12-07T01:22:40,997 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-1, size=369 (369bytes) 2024-12-07T01:22:40,997 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-07T01:22:40,998 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-07T01:22:40,998 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-1 2024-12-07T01:22:41,004 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-1 after 3ms 2024-12-07T01:22:41,011 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-1 took 16ms 2024-12-07T01:22:41,016 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-1 so closing down 2024-12-07T01:22:41,019 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-07T01:22:41,021 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000001-wal-1.temp 2024-12-07T01:22:41,022 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:41,022 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:41,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741841_1017 (size=369) 2024-12-07T01:22:41,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741841_1017 (size=369) 2024-12-07T01:22:41,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741841_1017 (size=369) 2024-12-07T01:22:41,036 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-07T01:22:41,040 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000002 2024-12-07T01:22:41,043 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-07T01:22:41,043 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-1, journal: Splitting hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-1, size=369 (369bytes) at 1733534560997Finishing writing output for hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-1 so closing down at 1733534561016 (+19 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000001-wal-1.temp at 1733534561021 (+5 ms)3 split writer threads finished at 1733534561022 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733534561036 (+14 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000002 at 1733534561040 (+4 ms)Processed 2 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733534561043 (+3 ms) 2024-12-07T01:22:41,062 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-2, size=231 (231bytes) 2024-12-07T01:22:41,062 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-2 2024-12-07T01:22:41,063 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-2 after 1ms 2024-12-07T01:22:41,066 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-2 took 5ms 2024-12-07T01:22:41,068 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-2 so closing down 2024-12-07T01:22:41,068 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:41,070 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-07T01:22:41,072 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000002-wal-2.temp 2024-12-07T01:22:41,072 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:41,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741842_1018 (size=231) 2024-12-07T01:22:41,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741842_1018 (size=231) 2024-12-07T01:22:41,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741842_1018 (size=231) 2024-12-07T01:22:41,081 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-07T01:22:41,087 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-12-07T01:22:41,089 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-07T01:22:41,089 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-2, journal: Splitting hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-2, size=231 (231bytes) at 1733534561062Finishing writing output for hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-2 so closing down at 1733534561068 (+6 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000002-wal-2.temp at 1733534561072 (+4 ms)3 split writer threads finished at 1733534561072Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733534561081 (+9 ms)Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733534561089 (+8 ms) 2024-12-07T01:22:41,089 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:41,092 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:41,103 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal.1733534561093, exclude list is [], retry=0 2024-12-07T01:22:41,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:41,107 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:41,107 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:41,109 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal.1733534561093 2024-12-07T01:22:41,110 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:34991:34991)] 2024-12-07T01:22:41,110 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 652f2506c256e0809ec156571eccb513, NAME => 'testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:41,110 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:41,110 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:41,110 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:41,112 INFO [StoreOpener-652f2506c256e0809ec156571eccb513-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:41,114 INFO [StoreOpener-652f2506c256e0809ec156571eccb513-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 652f2506c256e0809ec156571eccb513 columnFamilyName a 2024-12-07T01:22:41,114 DEBUG [StoreOpener-652f2506c256e0809ec156571eccb513-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:41,116 INFO [StoreOpener-652f2506c256e0809ec156571eccb513-1 {}] regionserver.HStore(327): Store=652f2506c256e0809ec156571eccb513/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:41,116 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:41,117 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513 2024-12-07T01:22:41,120 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513 2024-12-07T01:22:41,121 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000002 2024-12-07T01:22:41,129 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000002 2024-12-07T01:22:41,133 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 652f2506c256e0809ec156571eccb513 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-07T01:22:41,182 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/.tmp/a/eb12f0385ae3406abb34b009880f8bdb is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733534560934/Put/seqid=0 2024-12-07T01:22:41,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741844_1020 (size=5170) 2024-12-07T01:22:41,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741844_1020 (size=5170) 2024-12-07T01:22:41,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741844_1020 (size=5170) 2024-12-07T01:22:41,196 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/.tmp/a/eb12f0385ae3406abb34b009880f8bdb 2024-12-07T01:22:41,244 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/.tmp/a/eb12f0385ae3406abb34b009880f8bdb as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/a/eb12f0385ae3406abb34b009880f8bdb 2024-12-07T01:22:41,255 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/a/eb12f0385ae3406abb34b009880f8bdb, entries=2, sequenceid=2, filesize=5.0 K 2024-12-07T01:22:41,262 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 652f2506c256e0809ec156571eccb513 in 127ms, sequenceid=2, compaction requested=false; wal=null 2024-12-07T01:22:41,264 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/0000000000000000002 2024-12-07T01:22:41,265 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:41,265 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:41,269 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 652f2506c256e0809ec156571eccb513 2024-12-07T01:22:41,272 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/652f2506c256e0809ec156571eccb513/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-07T01:22:41,274 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 652f2506c256e0809ec156571eccb513; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69885341, jitterRate=0.04137273132801056}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:22:41,275 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 652f2506c256e0809ec156571eccb513: Writing region info on filesystem at 1733534561110Initializing all the Stores at 1733534561112 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534561112Obtaining lock to block concurrent updates at 1733534561133 (+21 ms)Preparing flush snapshotting stores in 652f2506c256e0809ec156571eccb513 at 1733534561133Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733534561135 (+2 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733534560851.652f2506c256e0809ec156571eccb513. at 1733534561136 (+1 ms)Flushing 652f2506c256e0809ec156571eccb513/a: creating writer at 1733534561137 (+1 ms)Flushing 652f2506c256e0809ec156571eccb513/a: appending metadata at 1733534561173 (+36 ms)Flushing 652f2506c256e0809ec156571eccb513/a: closing flushed file at 1733534561175 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b843f13: reopening flushed file at 1733534561242 (+67 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 652f2506c256e0809ec156571eccb513 in 127ms, sequenceid=2, compaction requested=false; wal=null at 1733534561262 (+20 ms)Cleaning up temporary data from old regions at 1733534561265 (+3 ms)Region opened successfully at 1733534561275 (+10 ms) 2024-12-07T01:22:41,303 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=380 (was 369) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:38188 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:37974 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:36673/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:44576 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:44702 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:58954 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:59074 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=365 (was 365), ProcessCount=11 (was 11), AvailableMemoryMB=5295 (was 5307) 2024-12-07T01:22:41,313 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=380, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=5294 2024-12-07T01:22:41,328 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:41,330 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:41,331 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:41,335 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-97201656, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/hregion-97201656, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:41,345 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-97201656/hregion-97201656.1733534561335, exclude list is [], retry=0 2024-12-07T01:22:41,349 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:41,350 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:41,350 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:41,353 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-97201656/hregion-97201656.1733534561335 2024-12-07T01:22:41,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627)] 2024-12-07T01:22:41,354 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a1c5e26b398d952aed2218dd15bd8dab, NAME => 'testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/hbase 2024-12-07T01:22:41,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741846_1022 (size=64) 2024-12-07T01:22:41,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741846_1022 (size=64) 2024-12-07T01:22:41,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741846_1022 (size=64) 2024-12-07T01:22:41,368 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:41,370 INFO [StoreOpener-a1c5e26b398d952aed2218dd15bd8dab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,372 INFO [StoreOpener-a1c5e26b398d952aed2218dd15bd8dab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1c5e26b398d952aed2218dd15bd8dab columnFamilyName a 2024-12-07T01:22:41,372 DEBUG [StoreOpener-a1c5e26b398d952aed2218dd15bd8dab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:41,373 INFO [StoreOpener-a1c5e26b398d952aed2218dd15bd8dab-1 {}] regionserver.HStore(327): Store=a1c5e26b398d952aed2218dd15bd8dab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:41,373 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,374 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,375 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,375 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,375 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,377 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,380 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:41,381 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a1c5e26b398d952aed2218dd15bd8dab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66955663, jitterRate=-0.002282872796058655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:22:41,381 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a1c5e26b398d952aed2218dd15bd8dab: Writing region info on filesystem at 1733534561368Initializing all the Stores at 1733534561370 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534561370Cleaning up temporary data from old regions at 1733534561375 (+5 ms)Region opened successfully at 1733534561381 (+6 ms) 2024-12-07T01:22:41,381 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a1c5e26b398d952aed2218dd15bd8dab, disabling compactions & flushes 2024-12-07T01:22:41,381 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab. 2024-12-07T01:22:41,381 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab. 2024-12-07T01:22:41,381 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab. after waiting 0 ms 2024-12-07T01:22:41,381 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab. 2024-12-07T01:22:41,382 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab. 2024-12-07T01:22:41,382 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a1c5e26b398d952aed2218dd15bd8dab: Waiting for close lock at 1733534561381Disabling compacts and flushes for region at 1733534561381Disabling writes for close at 1733534561381Writing region close event to WAL at 1733534561382 (+1 ms)Closed at 1733534561382 2024-12-07T01:22:41,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741845_1021 (size=93) 2024-12-07T01:22:41,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741845_1021 (size=93) 2024-12-07T01:22:41,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741845_1021 (size=93) 2024-12-07T01:22:41,387 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:22:41,387 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-97201656:(num 1733534561335) 2024-12-07T01:22:41,388 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-07T01:22:41,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741847_1023 (size=369) 2024-12-07T01:22:41,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741847_1023 (size=369) 2024-12-07T01:22:41,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741847_1023 (size=369) 2024-12-07T01:22:41,400 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-07T01:22:41,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741848_1024 (size=231) 2024-12-07T01:22:41,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741848_1024 (size=231) 2024-12-07T01:22:41,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741848_1024 (size=231) 2024-12-07T01:22:41,427 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-2, size=231 (231bytes) 2024-12-07T01:22:41,427 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-2 2024-12-07T01:22:41,428 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-2 after 1ms 2024-12-07T01:22:41,432 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-2 took 5ms 2024-12-07T01:22:41,434 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-2 so closing down 2024-12-07T01:22:41,434 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:41,436 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-07T01:22:41,437 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002-wal-2.temp 2024-12-07T01:22:41,438 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:41,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741849_1025 (size=231) 2024-12-07T01:22:41,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741849_1025 (size=231) 2024-12-07T01:22:41,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741849_1025 (size=231) 2024-12-07T01:22:41,446 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-07T01:22:41,449 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002 2024-12-07T01:22:41,449 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-07T01:22:41,449 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-2, journal: Splitting hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-2, size=231 (231bytes) at 1733534561427Finishing writing output for hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-2 so closing down at 1733534561434 (+7 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002-wal-2.temp at 1733534561438 (+4 ms)3 split writer threads finished at 1733534561438Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733534561446 (+8 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002 at 1733534561449 (+3 ms)Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733534561449 2024-12-07T01:22:41,464 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-1, size=369 (369bytes) 2024-12-07T01:22:41,464 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-1 2024-12-07T01:22:41,465 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-1 after 1ms 2024-12-07T01:22:41,468 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-1 took 4ms 2024-12-07T01:22:41,471 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-1 so closing down 2024-12-07T01:22:41,471 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:41,474 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-07T01:22:41,476 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000001-wal-1.temp 2024-12-07T01:22:41,476 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:41,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741850_1026 (size=369) 2024-12-07T01:22:41,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741850_1026 (size=369) 2024-12-07T01:22:41,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741850_1026 (size=369) 2024-12-07T01:22:41,486 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-07T01:22:41,492 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002, length=231 2024-12-07T01:22:41,494 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002 2024-12-07T01:22:41,495 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-07T01:22:41,495 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-1, journal: Splitting hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-1, size=369 (369bytes) at 1733534561464Finishing writing output for hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-1 so closing down at 1733534561471 (+7 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000001-wal-1.temp at 1733534561476 (+5 ms)3 split writer threads finished at 1733534561476Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733534561486 (+10 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002 at 1733534561495 (+9 ms)Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733534561495 2024-12-07T01:22:41,495 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:41,497 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:41,508 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal.1733534561498, exclude list is [], retry=0 2024-12-07T01:22:41,511 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:41,511 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:41,512 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:41,514 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal.1733534561498 2024-12-07T01:22:41,514 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627)] 2024-12-07T01:22:41,514 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a1c5e26b398d952aed2218dd15bd8dab, NAME => 'testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:41,515 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:41,515 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,515 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,516 INFO [StoreOpener-a1c5e26b398d952aed2218dd15bd8dab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,518 INFO [StoreOpener-a1c5e26b398d952aed2218dd15bd8dab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1c5e26b398d952aed2218dd15bd8dab columnFamilyName a 2024-12-07T01:22:41,518 DEBUG [StoreOpener-a1c5e26b398d952aed2218dd15bd8dab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:41,519 INFO [StoreOpener-a1c5e26b398d952aed2218dd15bd8dab-1 {}] regionserver.HStore(327): Store=a1c5e26b398d952aed2218dd15bd8dab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:41,519 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,520 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,521 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,522 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002 2024-12-07T01:22:41,526 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002 2024-12-07T01:22:41,526 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a1c5e26b398d952aed2218dd15bd8dab 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-07T01:22:41,541 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/.tmp/a/dd4297fd67c94e719a3b26dbe58b3824 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733534561388/Put/seqid=0 2024-12-07T01:22:41,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741852_1028 (size=5170) 2024-12-07T01:22:41,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741852_1028 (size=5170) 2024-12-07T01:22:41,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741852_1028 (size=5170) 2024-12-07T01:22:41,550 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/.tmp/a/dd4297fd67c94e719a3b26dbe58b3824 2024-12-07T01:22:41,558 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/.tmp/a/dd4297fd67c94e719a3b26dbe58b3824 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/a/dd4297fd67c94e719a3b26dbe58b3824 2024-12-07T01:22:41,567 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/a/dd4297fd67c94e719a3b26dbe58b3824, entries=2, sequenceid=2, filesize=5.0 K 2024-12-07T01:22:41,567 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for a1c5e26b398d952aed2218dd15bd8dab in 41ms, sequenceid=2, compaction requested=false; wal=null 2024-12-07T01:22:41,568 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/0000000000000000002 2024-12-07T01:22:41,569 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,569 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,572 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a1c5e26b398d952aed2218dd15bd8dab 2024-12-07T01:22:41,575 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/a1c5e26b398d952aed2218dd15bd8dab/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-07T01:22:41,576 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a1c5e26b398d952aed2218dd15bd8dab; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63922964, jitterRate=-0.04747360944747925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:22:41,577 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a1c5e26b398d952aed2218dd15bd8dab: Writing region info on filesystem at 1733534561515Initializing all the Stores at 1733534561516 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534561516Obtaining lock to block concurrent updates at 1733534561526 (+10 ms)Preparing flush snapshotting stores in a1c5e26b398d952aed2218dd15bd8dab at 1733534561526Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733534561526Flushing stores of testReplayEditsWrittenIntoWAL,,1733534561329.a1c5e26b398d952aed2218dd15bd8dab. at 1733534561526Flushing a1c5e26b398d952aed2218dd15bd8dab/a: creating writer at 1733534561526Flushing a1c5e26b398d952aed2218dd15bd8dab/a: appending metadata at 1733534561541 (+15 ms)Flushing a1c5e26b398d952aed2218dd15bd8dab/a: closing flushed file at 1733534561541Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32eb0ed0: reopening flushed file at 1733534561557 (+16 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for a1c5e26b398d952aed2218dd15bd8dab in 41ms, sequenceid=2, compaction requested=false; wal=null at 1733534561567 (+10 ms)Cleaning up temporary data from old regions at 1733534561569 (+2 ms)Region opened successfully at 1733534561577 (+8 ms) 2024-12-07T01:22:41,591 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=390 (was 380) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:59150 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:44778 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:37974 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:38234 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:58954 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=365 (was 365), ProcessCount=11 (was 11), AvailableMemoryMB=5289 (was 5294) 2024-12-07T01:22:41,602 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=390, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=5289 2024-12-07T01:22:41,619 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:41,621 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:41,622 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:41,625 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58253884, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/hregion-58253884, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:41,636 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58253884/hregion-58253884.1733534561626, exclude list is [], retry=0 2024-12-07T01:22:41,640 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:41,640 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:41,640 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:41,642 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58253884/hregion-58253884.1733534561626 2024-12-07T01:22:41,643 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:41,643 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2b0ad8c5a9da54fdf1e223e914f5972e, NAME => 'testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/hbase 2024-12-07T01:22:41,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741854_1030 (size=64) 2024-12-07T01:22:41,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741854_1030 (size=64) 2024-12-07T01:22:41,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741854_1030 (size=64) 2024-12-07T01:22:41,655 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:41,656 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:41,658 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b0ad8c5a9da54fdf1e223e914f5972e columnFamilyName a 2024-12-07T01:22:41,658 DEBUG [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:41,659 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(327): Store=2b0ad8c5a9da54fdf1e223e914f5972e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:41,659 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:41,661 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b0ad8c5a9da54fdf1e223e914f5972e columnFamilyName b 2024-12-07T01:22:41,661 DEBUG [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:41,661 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(327): Store=2b0ad8c5a9da54fdf1e223e914f5972e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:41,661 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:41,663 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b0ad8c5a9da54fdf1e223e914f5972e columnFamilyName c 2024-12-07T01:22:41,663 DEBUG [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:41,664 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(327): Store=2b0ad8c5a9da54fdf1e223e914f5972e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:41,664 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:41,665 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:41,665 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:41,667 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:41,667 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:41,667 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:41,668 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:41,671 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:41,672 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2b0ad8c5a9da54fdf1e223e914f5972e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74496284, jitterRate=0.11008113622665405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:41,672 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2b0ad8c5a9da54fdf1e223e914f5972e: Writing region info on filesystem at 1733534561655Initializing all the Stores at 1733534561656 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534561656Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534561656Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534561656Cleaning up temporary data from old regions at 1733534561667 (+11 ms)Region opened successfully at 1733534561672 (+5 ms) 2024-12-07T01:22:41,673 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2b0ad8c5a9da54fdf1e223e914f5972e, disabling compactions & flushes 2024-12-07T01:22:41,673 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. 2024-12-07T01:22:41,673 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. 2024-12-07T01:22:41,673 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. after waiting 0 ms 2024-12-07T01:22:41,673 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. 2024-12-07T01:22:41,674 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. 2024-12-07T01:22:41,674 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2b0ad8c5a9da54fdf1e223e914f5972e: Waiting for close lock at 1733534561673Disabling compacts and flushes for region at 1733534561673Disabling writes for close at 1733534561673Writing region close event to WAL at 1733534561674 (+1 ms)Closed at 1733534561674 2024-12-07T01:22:41,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741853_1029 (size=93) 2024-12-07T01:22:41,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741853_1029 (size=93) 2024-12-07T01:22:41,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741853_1029 (size=93) 2024-12-07T01:22:41,681 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:22:41,681 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58253884:(num 1733534561626) 2024-12-07T01:22:41,681 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:41,684 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:41,699 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684, exclude list is [], retry=0 2024-12-07T01:22:41,702 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:41,703 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:41,704 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:41,706 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 2024-12-07T01:22:41,707 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627)] 2024-12-07T01:22:41,896 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684, size=0 (0bytes) 2024-12-07T01:22:41,896 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 might be still open, length is 0 2024-12-07T01:22:41,896 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 2024-12-07T01:22:41,897 WARN [IPC Server handler 3 on default port 36673 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-07T01:22:41,897 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 after 1ms 2024-12-07T01:22:42,518 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:44806 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46343:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44806 dst: /127.0.0.1:46343 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46343 remote=/127.0.0.1:44806]. Total timeout mills is 60000, 59346 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:42,520 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:59180 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45293:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59180 dst: /127.0.0.1:45293 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:42,520 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:38286 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33463:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38286 dst: /127.0.0.1:33463 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:42,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741855_1032 (size=470675) 2024-12-07T01:22:42,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741855_1032 (size=470675) 2024-12-07T01:22:42,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741855_1032 (size=470675) 2024-12-07T01:22:45,899 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 after 4003ms 2024-12-07T01:22:45,908 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 took 4013ms 2024-12-07T01:22:45,914 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733534561684.temp 2024-12-07T01:22:45,921 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000000001-wal.1733534561684.temp 2024-12-07T01:22:45,941 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T01:22:45,985 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684; continuing. 2024-12-07T01:22:45,985 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 so closing down 2024-12-07T01:22:45,985 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:45,986 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:45,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741856_1033 (size=470683) 2024-12-07T01:22:45,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741856_1033 (size=470683) 2024-12-07T01:22:45,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741856_1033 (size=470683) 2024-12-07T01:22:45,993 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000000001-wal.1733534561684.temp (wrote 3002 edits, skipped 0 edits in 27 ms) 2024-12-07T01:22:45,995 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000000001-wal.1733534561684.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000003002 2024-12-07T01:22:45,995 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 86 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684, size=0, length=0, corrupted=false, cancelled=false 2024-12-07T01:22:45,995 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684, journal: Splitting hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684, size=0 (0bytes) at 1733534561896Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000000001-wal.1733534561684.temp at 1733534565921 (+4025 ms)Split 1024 edits, skipped 0 edits. at 1733534565955 (+34 ms)Split 2048 edits, skipped 0 edits. at 1733534565971 (+16 ms)Finishing writing output for hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 so closing down at 1733534565985 (+14 ms)3 split writer threads finished at 1733534565986 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000000001-wal.1733534561684.temp (wrote 3002 edits, skipped 0 edits in 27 ms) at 1733534565993 (+7 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000000001-wal.1733534561684.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000003002 at 1733534565995 (+2 ms)Processed 3002 edits across 1 Regions in 86 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684, size=0, length=0, corrupted=false, cancelled=false at 1733534565995 2024-12-07T01:22:45,998 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 to hdfs://localhost:36673/hbase/oldWALs/wal.1733534561684 2024-12-07T01:22:46,000 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000003002 2024-12-07T01:22:46,000 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:46,003 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:46,011 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-07T01:22:46,016 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534566004, exclude list is [], retry=0 2024-12-07T01:22:46,020 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:46,020 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:46,021 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:46,024 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534566004 2024-12-07T01:22:46,024 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:34991:34991)] 2024-12-07T01:22:46,024 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:46,026 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:46,027 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b0ad8c5a9da54fdf1e223e914f5972e columnFamilyName a 2024-12-07T01:22:46,028 DEBUG [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:46,028 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(327): Store=2b0ad8c5a9da54fdf1e223e914f5972e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:46,029 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:46,030 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b0ad8c5a9da54fdf1e223e914f5972e columnFamilyName b 2024-12-07T01:22:46,030 DEBUG [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:46,031 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(327): Store=2b0ad8c5a9da54fdf1e223e914f5972e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:46,031 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:46,032 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b0ad8c5a9da54fdf1e223e914f5972e columnFamilyName c 2024-12-07T01:22:46,032 DEBUG [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:46,033 INFO [StoreOpener-2b0ad8c5a9da54fdf1e223e914f5972e-1 {}] regionserver.HStore(327): Store=2b0ad8c5a9da54fdf1e223e914f5972e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:46,033 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:46,035 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:46,037 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:46,039 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000003002 2024-12-07T01:22:46,079 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-07T01:22:46,411 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2b0ad8c5a9da54fdf1e223e914f5972e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-07T01:22:46,445 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/a/effe79bddd4c4e8c9a886f05e48ffd4e is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733534561719/Put/seqid=0 2024-12-07T01:22:46,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741858_1035 (size=50463) 2024-12-07T01:22:46,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741858_1035 (size=50463) 2024-12-07T01:22:46,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741858_1035 (size=50463) 2024-12-07T01:22:46,454 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/a/effe79bddd4c4e8c9a886f05e48ffd4e 2024-12-07T01:22:46,462 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/a/effe79bddd4c4e8c9a886f05e48ffd4e as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/a/effe79bddd4c4e8c9a886f05e48ffd4e 2024-12-07T01:22:46,469 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/a/effe79bddd4c4e8c9a886f05e48ffd4e, entries=754, sequenceid=754, filesize=49.3 K 2024-12-07T01:22:46,469 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 2b0ad8c5a9da54fdf1e223e914f5972e in 59ms, sequenceid=754, compaction requested=false; wal=null 2024-12-07T01:22:46,486 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-07T01:22:46,486 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2b0ad8c5a9da54fdf1e223e914f5972e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-07T01:22:46,495 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/a/194a8b982b974f1e93243c7055adda03 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733534561762/Put/seqid=0 2024-12-07T01:22:46,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741859_1036 (size=20072) 2024-12-07T01:22:46,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741859_1036 (size=20072) 2024-12-07T01:22:46,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741859_1036 (size=20072) 2024-12-07T01:22:46,504 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/a/194a8b982b974f1e93243c7055adda03 2024-12-07T01:22:46,528 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/b/197ceb68b85443bf85282bce0bfb9f07 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733534561780/Put/seqid=0 2024-12-07T01:22:46,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741860_1037 (size=35835) 2024-12-07T01:22:46,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741860_1037 (size=35835) 2024-12-07T01:22:46,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741860_1037 (size=35835) 2024-12-07T01:22:46,537 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/b/197ceb68b85443bf85282bce0bfb9f07 2024-12-07T01:22:46,544 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/a/194a8b982b974f1e93243c7055adda03 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/a/194a8b982b974f1e93243c7055adda03 2024-12-07T01:22:46,550 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/a/194a8b982b974f1e93243c7055adda03, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-07T01:22:46,552 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/b/197ceb68b85443bf85282bce0bfb9f07 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/b/197ceb68b85443bf85282bce0bfb9f07 2024-12-07T01:22:46,558 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/b/197ceb68b85443bf85282bce0bfb9f07, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-07T01:22:46,558 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 2b0ad8c5a9da54fdf1e223e914f5972e in 72ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-07T01:22:46,572 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-07T01:22:46,573 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2b0ad8c5a9da54fdf1e223e914f5972e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-07T01:22:46,581 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/b/6371ec4cdbfa4fc2bd4ba590272edfe0 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733534561796/Put/seqid=0 2024-12-07T01:22:46,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741861_1038 (size=35082) 2024-12-07T01:22:46,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741861_1038 (size=35082) 2024-12-07T01:22:46,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741861_1038 (size=35082) 2024-12-07T01:22:46,590 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/b/6371ec4cdbfa4fc2bd4ba590272edfe0 2024-12-07T01:22:46,610 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/c/30383731e405422b9cc8136f1865fc6b is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733534561821/Put/seqid=0 2024-12-07T01:22:46,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741862_1039 (size=20825) 2024-12-07T01:22:46,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741862_1039 (size=20825) 2024-12-07T01:22:46,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741862_1039 (size=20825) 2024-12-07T01:22:46,618 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/c/30383731e405422b9cc8136f1865fc6b 2024-12-07T01:22:46,625 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/b/6371ec4cdbfa4fc2bd4ba590272edfe0 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/b/6371ec4cdbfa4fc2bd4ba590272edfe0 2024-12-07T01:22:46,632 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/b/6371ec4cdbfa4fc2bd4ba590272edfe0, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-07T01:22:46,633 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/c/30383731e405422b9cc8136f1865fc6b as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/c/30383731e405422b9cc8136f1865fc6b 2024-12-07T01:22:46,640 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/c/30383731e405422b9cc8136f1865fc6b, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-07T01:22:46,640 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 2b0ad8c5a9da54fdf1e223e914f5972e in 68ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-07T01:22:46,647 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733534561863/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. 2024-12-07T01:22:46,649 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000003002 2024-12-07T01:22:46,650 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-07T01:22:46,650 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2b0ad8c5a9da54fdf1e223e914f5972e 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-07T01:22:46,657 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/c/c886df529ce84593b15417a794bd8097 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733534561828/Put/seqid=0 2024-12-07T01:22:46,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741863_1040 (size=50301) 2024-12-07T01:22:46,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741863_1040 (size=50301) 2024-12-07T01:22:46,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741863_1040 (size=50301) 2024-12-07T01:22:46,668 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/c/c886df529ce84593b15417a794bd8097 2024-12-07T01:22:46,674 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c886df529ce84593b15417a794bd8097 2024-12-07T01:22:46,675 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/.tmp/c/c886df529ce84593b15417a794bd8097 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/c/c886df529ce84593b15417a794bd8097 2024-12-07T01:22:46,682 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c886df529ce84593b15417a794bd8097 2024-12-07T01:22:46,682 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/c/c886df529ce84593b15417a794bd8097, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-07T01:22:46,682 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 2b0ad8c5a9da54fdf1e223e914f5972e in 32ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-07T01:22:46,683 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/0000000000000003002 2024-12-07T01:22:46,684 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:46,684 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:46,685 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T01:22:46,687 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2b0ad8c5a9da54fdf1e223e914f5972e 2024-12-07T01:22:46,689 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenIntoWAL/2b0ad8c5a9da54fdf1e223e914f5972e/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-07T01:22:46,691 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2b0ad8c5a9da54fdf1e223e914f5972e; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70258480, jitterRate=0.04693293571472168}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T01:22:46,691 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2b0ad8c5a9da54fdf1e223e914f5972e: Writing region info on filesystem at 1733534566024Initializing all the Stores at 1733534566026 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534566026Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534566026Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534566026Cleaning up temporary data from old regions at 1733534566684 (+658 ms)Region opened successfully at 1733534566691 (+7 ms) 2024-12-07T01:22:46,754 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2b0ad8c5a9da54fdf1e223e914f5972e, disabling compactions & flushes 2024-12-07T01:22:46,754 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. 2024-12-07T01:22:46,754 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. 2024-12-07T01:22:46,754 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. after waiting 0 ms 2024-12-07T01:22:46,755 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. 2024-12-07T01:22:46,763 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733534561619.2b0ad8c5a9da54fdf1e223e914f5972e. 2024-12-07T01:22:46,763 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2b0ad8c5a9da54fdf1e223e914f5972e: Waiting for close lock at 1733534566754Disabling compacts and flushes for region at 1733534566754Disabling writes for close at 1733534566754Writing region close event to WAL at 1733534566763 (+9 ms)Closed at 1733534566763 2024-12-07T01:22:46,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741857_1034 (size=93) 2024-12-07T01:22:46,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741857_1034 (size=93) 2024-12-07T01:22:46,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741857_1034 (size=93) 2024-12-07T01:22:46,770 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:22:46,770 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733534566004) 2024-12-07T01:22:46,783 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=404 (was 390) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-611872767_22 at /127.0.0.1:46908 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:33055 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:36673 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-611872767_22 at /127.0.0.1:36676 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-611872767_22 at /127.0.0.1:36706 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43609 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-611872767_22 at /127.0.0.1:40730 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:36673 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33055 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:43609 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 365), ProcessCount=11 (was 11), AvailableMemoryMB=5228 (was 5289) 2024-12-07T01:22:46,796 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5227 2024-12-07T01:22:46,810 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:46,813 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:46,814 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:46,817 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26287942, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/hregion-26287942, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:46,836 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26287942/hregion-26287942.1733534566818, exclude list is [], retry=0 2024-12-07T01:22:46,840 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:46,840 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:46,840 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:46,843 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26287942/hregion-26287942.1733534566818 2024-12-07T01:22:46,843 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:46,843 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ab33f4717318d35d22546b8e9d2f02ec, NAME => 'test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/hbase 2024-12-07T01:22:46,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741865_1042 (size=43) 2024-12-07T01:22:46,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741865_1042 (size=43) 2024-12-07T01:22:46,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741865_1042 (size=43) 2024-12-07T01:22:46,859 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:46,864 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:46,866 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab33f4717318d35d22546b8e9d2f02ec columnFamilyName a 2024-12-07T01:22:46,866 DEBUG [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:46,867 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(327): Store=ab33f4717318d35d22546b8e9d2f02ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:46,867 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:46,869 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab33f4717318d35d22546b8e9d2f02ec columnFamilyName b 2024-12-07T01:22:46,869 DEBUG [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:46,869 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(327): Store=ab33f4717318d35d22546b8e9d2f02ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:46,870 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:46,871 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab33f4717318d35d22546b8e9d2f02ec columnFamilyName c 2024-12-07T01:22:46,871 DEBUG [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:46,872 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(327): Store=ab33f4717318d35d22546b8e9d2f02ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:46,872 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:46,873 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:46,873 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:46,875 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:46,875 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:46,876 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:46,877 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:46,880 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:46,881 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab33f4717318d35d22546b8e9d2f02ec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63597837, jitterRate=-0.052318379282951355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:46,882 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab33f4717318d35d22546b8e9d2f02ec: Writing region info on filesystem at 1733534566859Initializing all the Stores at 1733534566860 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534566861 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534566863 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534566863Cleaning up temporary data from old regions at 1733534566875 (+12 ms)Region opened successfully at 1733534566882 (+7 ms) 2024-12-07T01:22:46,882 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab33f4717318d35d22546b8e9d2f02ec, disabling compactions & flushes 2024-12-07T01:22:46,882 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. 2024-12-07T01:22:46,882 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. 2024-12-07T01:22:46,882 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. after waiting 0 ms 2024-12-07T01:22:46,882 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. 2024-12-07T01:22:46,883 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. 2024-12-07T01:22:46,883 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab33f4717318d35d22546b8e9d2f02ec: Waiting for close lock at 1733534566882Disabling compacts and flushes for region at 1733534566882Disabling writes for close at 1733534566882Writing region close event to WAL at 1733534566883 (+1 ms)Closed at 1733534566883 2024-12-07T01:22:46,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741864_1041 (size=93) 2024-12-07T01:22:46,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741864_1041 (size=93) 2024-12-07T01:22:46,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741864_1041 (size=93) 2024-12-07T01:22:46,890 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:22:46,890 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26287942:(num 1733534566818) 2024-12-07T01:22:46,890 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:46,893 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:46,905 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893, exclude list is [], retry=0 2024-12-07T01:22:46,908 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:46,908 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:46,909 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:46,911 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893 2024-12-07T01:22:46,912 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627)] 2024-12-07T01:22:47,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741866_1043 (size=344306) 2024-12-07T01:22:47,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741866_1043 (size=344306) 2024-12-07T01:22:47,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741866_1043 (size=344306) 2024-12-07T01:22:47,107 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893, size=336.2 K (344306bytes) 2024-12-07T01:22:47,108 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893 2024-12-07T01:22:47,108 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893 after 0ms 2024-12-07T01:22:47,113 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893 took 6ms 2024-12-07T01:22:47,118 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733534566893.temp 2024-12-07T01:22:47,120 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000000001-wal.1733534566893.temp 2024-12-07T01:22:47,149 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893 so closing down 2024-12-07T01:22:47,149 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:47,149 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:47,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741867_1044 (size=344306) 2024-12-07T01:22:47,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741867_1044 (size=344306) 2024-12-07T01:22:47,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741867_1044 (size=344306) 2024-12-07T01:22:47,155 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000000001-wal.1733534566893.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-12-07T01:22:47,157 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000000001-wal.1733534566893.temp to hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003000 2024-12-07T01:22:47,157 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-12-07T01:22:47,157 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893, journal: Splitting hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893, size=336.2 K (344306bytes) at 1733534567107Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000000001-wal.1733534566893.temp at 1733534567120 (+13 ms)Split 1024 edits, skipped 0 edits. at 1733534567127 (+7 ms)Split 2048 edits, skipped 0 edits. at 1733534567139 (+12 ms)Finishing writing output for hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893 so closing down at 1733534567149 (+10 ms)3 split writer threads finished at 1733534567149Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000000001-wal.1733534566893.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1733534567155 (+6 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000000001-wal.1733534566893.temp to hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003000 at 1733534567157 (+2 ms)Processed 3000 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1733534567157 2024-12-07T01:22:47,159 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534566893 to hdfs://localhost:36673/hbase/oldWALs/wal.1733534566893 2024-12-07T01:22:47,160 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003000 2024-12-07T01:22:47,161 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:47,163 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:47,182 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164, exclude list is [], retry=0 2024-12-07T01:22:47,185 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:47,185 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:47,186 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:47,188 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164 2024-12-07T01:22:47,188 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:47,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741868_1045 (size=344433) 2024-12-07T01:22:47,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741868_1045 (size=344433) 2024-12-07T01:22:47,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741868_1045 (size=344433) 2024-12-07T01:22:47,379 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164, size=336.4 K (344433bytes) 2024-12-07T01:22:47,379 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164 2024-12-07T01:22:47,379 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164 after 0ms 2024-12-07T01:22:47,384 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164 took 6ms 2024-12-07T01:22:47,387 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733534567164.temp 2024-12-07T01:22:47,389 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003001-wal.1733534567164.temp 2024-12-07T01:22:47,417 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164 so closing down 2024-12-07T01:22:47,417 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:47,417 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:47,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741869_1046 (size=344433) 2024-12-07T01:22:47,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741869_1046 (size=344433) 2024-12-07T01:22:47,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741869_1046 (size=344433) 2024-12-07T01:22:47,427 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003001-wal.1733534567164.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-12-07T01:22:47,429 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003001-wal.1733534567164.temp to hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000006000 2024-12-07T01:22:47,430 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 45 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-12-07T01:22:47,430 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164, journal: Splitting hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164, size=336.4 K (344433bytes) at 1733534567379Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003001-wal.1733534567164.temp at 1733534567389 (+10 ms)Split 1024 edits, skipped 0 edits. at 1733534567397 (+8 ms)Split 2048 edits, skipped 0 edits. at 1733534567407 (+10 ms)Finishing writing output for hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164 so closing down at 1733534567417 (+10 ms)3 split writer threads finished at 1733534567417Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003001-wal.1733534567164.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1733534567427 (+10 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003001-wal.1733534567164.temp to hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000006000 at 1733534567429 (+2 ms)Processed 3000 edits across 1 Regions in 45 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1733534567430 (+1 ms) 2024-12-07T01:22:47,432 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567164 to hdfs://localhost:36673/hbase/oldWALs/wal.1733534567164 2024-12-07T01:22:47,433 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000006000 2024-12-07T01:22:47,433 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:47,435 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/test2727-manual,16010,1733534566809, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:47,450 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567436, exclude list is [], retry=0 2024-12-07T01:22:47,453 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:47,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:47,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:47,457 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733534566809/wal.1733534567436 2024-12-07T01:22:47,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:47,458 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ab33f4717318d35d22546b8e9d2f02ec, NAME => 'test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:47,458 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:47,458 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,458 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,460 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,461 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab33f4717318d35d22546b8e9d2f02ec columnFamilyName a 2024-12-07T01:22:47,462 DEBUG [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:47,462 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(327): Store=ab33f4717318d35d22546b8e9d2f02ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:47,463 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,464 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab33f4717318d35d22546b8e9d2f02ec columnFamilyName b 2024-12-07T01:22:47,464 DEBUG [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:47,465 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(327): Store=ab33f4717318d35d22546b8e9d2f02ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:47,465 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,466 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab33f4717318d35d22546b8e9d2f02ec columnFamilyName c 2024-12-07T01:22:47,466 DEBUG [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:47,467 INFO [StoreOpener-ab33f4717318d35d22546b8e9d2f02ec-1 {}] regionserver.HStore(327): Store=ab33f4717318d35d22546b8e9d2f02ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:47,467 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,468 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,472 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003000 2024-12-07T01:22:47,505 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003000 2024-12-07T01:22:47,507 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000006000 2024-12-07T01:22:47,539 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000006000 2024-12-07T01:22:47,540 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab33f4717318d35d22546b8e9d2f02ec 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-07T01:22:47,558 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/.tmp/a/101e91ebf8184001932c651566d97511 is 41, key is test2727/a:100/1733534567194/Put/seqid=0 2024-12-07T01:22:47,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741871_1048 (size=84227) 2024-12-07T01:22:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741871_1048 (size=84227) 2024-12-07T01:22:47,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741871_1048 (size=84227) 2024-12-07T01:22:47,567 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/.tmp/a/101e91ebf8184001932c651566d97511 2024-12-07T01:22:47,594 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/.tmp/b/6c49778ba7a84c40a8fdd440c7f2a9c0 is 41, key is test2727/b:100/1733534567256/Put/seqid=0 2024-12-07T01:22:47,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741872_1049 (size=84609) 2024-12-07T01:22:47,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741872_1049 (size=84609) 2024-12-07T01:22:47,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741872_1049 (size=84609) 2024-12-07T01:22:47,601 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/.tmp/b/6c49778ba7a84c40a8fdd440c7f2a9c0 2024-12-07T01:22:47,626 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/.tmp/c/7b095eb10826467498357e9cd3871410 is 41, key is test2727/c:100/1733534567312/Put/seqid=0 2024-12-07T01:22:47,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741873_1050 (size=84609) 2024-12-07T01:22:47,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741873_1050 (size=84609) 2024-12-07T01:22:47,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741873_1050 (size=84609) 2024-12-07T01:22:47,634 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/.tmp/c/7b095eb10826467498357e9cd3871410 2024-12-07T01:22:47,642 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/.tmp/a/101e91ebf8184001932c651566d97511 as hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/a/101e91ebf8184001932c651566d97511 2024-12-07T01:22:47,650 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/a/101e91ebf8184001932c651566d97511, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-07T01:22:47,651 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/.tmp/b/6c49778ba7a84c40a8fdd440c7f2a9c0 as hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/b/6c49778ba7a84c40a8fdd440c7f2a9c0 2024-12-07T01:22:47,659 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/b/6c49778ba7a84c40a8fdd440c7f2a9c0, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-07T01:22:47,661 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/.tmp/c/7b095eb10826467498357e9cd3871410 as hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/c/7b095eb10826467498357e9cd3871410 2024-12-07T01:22:47,669 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/c/7b095eb10826467498357e9cd3871410, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-07T01:22:47,669 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for ab33f4717318d35d22546b8e9d2f02ec in 130ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-07T01:22:47,671 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000003000 2024-12-07T01:22:47,672 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/0000000000000006000 2024-12-07T01:22:47,673 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,673 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,674 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:47,676 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab33f4717318d35d22546b8e9d2f02ec 2024-12-07T01:22:47,679 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/test2727/ab33f4717318d35d22546b8e9d2f02ec/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-07T01:22:47,680 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab33f4717318d35d22546b8e9d2f02ec; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59317234, jitterRate=-0.11610433459281921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:47,681 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab33f4717318d35d22546b8e9d2f02ec: Writing region info on filesystem at 1733534567458Initializing all the Stores at 1733534567460 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534567460Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534567460Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534567460Obtaining lock to block concurrent updates at 1733534567540 (+80 ms)Preparing flush snapshotting stores in ab33f4717318d35d22546b8e9d2f02ec at 1733534567540Finished memstore snapshotting test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733534567540Flushing stores of test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. at 1733534567540Flushing ab33f4717318d35d22546b8e9d2f02ec/a: creating writer at 1733534567540Flushing ab33f4717318d35d22546b8e9d2f02ec/a: appending metadata at 1733534567558 (+18 ms)Flushing ab33f4717318d35d22546b8e9d2f02ec/a: closing flushed file at 1733534567558Flushing ab33f4717318d35d22546b8e9d2f02ec/b: creating writer at 1733534567574 (+16 ms)Flushing ab33f4717318d35d22546b8e9d2f02ec/b: appending metadata at 1733534567592 (+18 ms)Flushing ab33f4717318d35d22546b8e9d2f02ec/b: closing flushed file at 1733534567592Flushing ab33f4717318d35d22546b8e9d2f02ec/c: creating writer at 1733534567608 (+16 ms)Flushing ab33f4717318d35d22546b8e9d2f02ec/c: appending metadata at 1733534567626 (+18 ms)Flushing ab33f4717318d35d22546b8e9d2f02ec/c: closing flushed file at 1733534567626Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30d24925: reopening flushed file at 1733534567641 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@419f6c41: reopening flushed file at 1733534567650 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42235a12: reopening flushed file at 1733534567660 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for ab33f4717318d35d22546b8e9d2f02ec in 130ms, sequenceid=6000, compaction requested=false; wal=null at 1733534567669 (+9 ms)Cleaning up temporary data from old regions at 1733534567673 (+4 ms)Region opened successfully at 1733534567681 (+8 ms) 2024-12-07T01:22:47,683 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-07T01:22:47,683 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab33f4717318d35d22546b8e9d2f02ec, disabling compactions & flushes 2024-12-07T01:22:47,683 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. 2024-12-07T01:22:47,683 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. 2024-12-07T01:22:47,683 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. after waiting 0 ms 2024-12-07T01:22:47,683 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. 2024-12-07T01:22:47,684 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733534566811.ab33f4717318d35d22546b8e9d2f02ec. 2024-12-07T01:22:47,685 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab33f4717318d35d22546b8e9d2f02ec: Waiting for close lock at 1733534567683Disabling compacts and flushes for region at 1733534567683Disabling writes for close at 1733534567683Writing region close event to WAL at 1733534567684 (+1 ms)Closed at 1733534567684 2024-12-07T01:22:47,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741870_1047 (size=93) 2024-12-07T01:22:47,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741870_1047 (size=93) 2024-12-07T01:22:47,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741870_1047 (size=93) 2024-12-07T01:22:47,691 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:22:47,691 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733534567436) 2024-12-07T01:22:47,702 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=406 (was 404) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:47034 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:36706 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:40828 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5052 (was 5227) 2024-12-07T01:22:47,713 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=406, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5051 2024-12-07T01:22:47,727 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:47,731 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-07T01:22:47,731 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-07T01:22:47,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T01:22:47,734 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-07T01:22:47,734 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:47,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-07T01:22:47,735 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-07T01:22:47,735 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733534567735 2024-12-07T01:22:47,736 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-07T01:22:47,737 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-07T01:22:47,745 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735 2024-12-07T01:22:47,751 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991)] 2024-12-07T01:22:47,755 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d82547b1fee89b4d5c9739086070661f, NAME => 'testSequentialEditLogSeqNum,,1733534567728.d82547b1fee89b4d5c9739086070661f.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:47,755 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733534567728.d82547b1fee89b4d5c9739086070661f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:47,755 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d82547b1fee89b4d5c9739086070661f 2024-12-07T01:22:47,755 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d82547b1fee89b4d5c9739086070661f 2024-12-07T01:22:47,756 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f doesn't exist for region: d82547b1fee89b4d5c9739086070661f on table testSequentialEditLogSeqNum 2024-12-07T01:22:47,757 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: d82547b1fee89b4d5c9739086070661f on table testSequentialEditLogSeqNum 2024-12-07T01:22:47,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741875_1052 (size=62) 2024-12-07T01:22:47,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741875_1052 (size=62) 2024-12-07T01:22:47,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741875_1052 (size=62) 2024-12-07T01:22:47,767 INFO [StoreOpener-d82547b1fee89b4d5c9739086070661f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d82547b1fee89b4d5c9739086070661f 2024-12-07T01:22:47,769 INFO [StoreOpener-d82547b1fee89b4d5c9739086070661f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82547b1fee89b4d5c9739086070661f columnFamilyName a 2024-12-07T01:22:47,769 DEBUG [StoreOpener-d82547b1fee89b4d5c9739086070661f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:47,769 INFO [StoreOpener-d82547b1fee89b4d5c9739086070661f-1 {}] regionserver.HStore(327): Store=d82547b1fee89b4d5c9739086070661f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:47,770 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d82547b1fee89b4d5c9739086070661f 2024-12-07T01:22:47,770 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f 2024-12-07T01:22:47,771 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f 2024-12-07T01:22:47,771 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d82547b1fee89b4d5c9739086070661f 2024-12-07T01:22:47,771 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d82547b1fee89b4d5c9739086070661f 2024-12-07T01:22:47,773 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d82547b1fee89b4d5c9739086070661f 2024-12-07T01:22:47,775 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:47,775 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d82547b1fee89b4d5c9739086070661f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72938553, jitterRate=0.08686913549900055}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:22:47,776 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d82547b1fee89b4d5c9739086070661f: Writing region info on filesystem at 1733534567755Initializing all the Stores at 1733534567767 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534567767Cleaning up temporary data from old regions at 1733534567771 (+4 ms)Region opened successfully at 1733534567776 (+5 ms) 2024-12-07T01:22:47,788 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d82547b1fee89b4d5c9739086070661f 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-07T01:22:47,813 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/.tmp/a/b2451babb29047f9a8a73079a6ac1185 is 81, key is testSequentialEditLogSeqNum/a:x0/1733534567776/Put/seqid=0 2024-12-07T01:22:47,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741876_1053 (size=5833) 2024-12-07T01:22:47,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741876_1053 (size=5833) 2024-12-07T01:22:47,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741876_1053 (size=5833) 2024-12-07T01:22:47,820 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/.tmp/a/b2451babb29047f9a8a73079a6ac1185 2024-12-07T01:22:47,827 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/.tmp/a/b2451babb29047f9a8a73079a6ac1185 as hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/a/b2451babb29047f9a8a73079a6ac1185 2024-12-07T01:22:47,833 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/a/b2451babb29047f9a8a73079a6ac1185, entries=10, sequenceid=13, filesize=5.7 K 2024-12-07T01:22:47,834 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d82547b1fee89b4d5c9739086070661f in 46ms, sequenceid=13, compaction requested=false 2024-12-07T01:22:47,834 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d82547b1fee89b4d5c9739086070661f: 2024-12-07T01:22:47,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T01:22:47,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T01:22:47,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T01:22:47,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T01:22:47,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-07T01:22:47,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741874_1051 (size=3017) 2024-12-07T01:22:47,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741874_1051 (size=3017) 2024-12-07T01:22:47,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741874_1051 (size=3017) 2024-12-07T01:22:47,858 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735, size=2.9 K (3017bytes) 2024-12-07T01:22:47,858 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735 2024-12-07T01:22:47,859 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735 after 1ms 2024-12-07T01:22:47,862 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735 took 4ms 2024-12-07T01:22:47,864 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735 so closing down 2024-12-07T01:22:47,864 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:47,865 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733534567735.temp 2024-12-07T01:22:47,867 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/recovered.edits/0000000000000000003-wal.1733534567735.temp 2024-12-07T01:22:47,867 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:47,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741877_1054 (size=2433) 2024-12-07T01:22:47,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741877_1054 (size=2433) 2024-12-07T01:22:47,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741877_1054 (size=2433) 2024-12-07T01:22:47,879 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/recovered.edits/0000000000000000003-wal.1733534567735.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-07T01:22:47,881 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/recovered.edits/0000000000000000003-wal.1733534567735.temp to hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/recovered.edits/0000000000000000020 2024-12-07T01:22:47,881 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-12-07T01:22:47,881 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735, journal: Splitting hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735, size=2.9 K (3017bytes) at 1733534567858Finishing writing output for hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735 so closing down at 1733534567864 (+6 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/recovered.edits/0000000000000000003-wal.1733534567735.temp at 1733534567867 (+3 ms)3 split writer threads finished at 1733534567867Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/recovered.edits/0000000000000000003-wal.1733534567735.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733534567879 (+12 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/recovered.edits/0000000000000000003-wal.1733534567735.temp to hdfs://localhost:36673/hbase/data/default/testSequentialEditLogSeqNum/d82547b1fee89b4d5c9739086070661f/recovered.edits/0000000000000000020 at 1733534567881 (+2 ms)Processed 17 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:36673/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733534567726/wal.1733534567735, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1733534567881 2024-12-07T01:22:47,894 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=411 (was 406) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:36706 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5033 (was 5051) 2024-12-07T01:22:47,906 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5032 2024-12-07T01:22:47,920 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:47,923 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:47,945 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:47,948 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23444869, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/hregion-23444869, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:47,969 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23444869/hregion-23444869.1733534567948, exclude list is [], retry=0 2024-12-07T01:22:47,973 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:47,973 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:47,974 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:47,975 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23444869/hregion-23444869.1733534567948 2024-12-07T01:22:47,976 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:47,976 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8bd8975e71cd1de4115532ce77199f0e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/hbase 2024-12-07T01:22:47,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741879_1056 (size=70) 2024-12-07T01:22:47,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741879_1056 (size=70) 2024-12-07T01:22:47,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741879_1056 (size=70) 2024-12-07T01:22:47,987 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:47,988 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:47,990 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bd8975e71cd1de4115532ce77199f0e columnFamilyName a 2024-12-07T01:22:47,990 DEBUG [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:47,991 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(327): Store=8bd8975e71cd1de4115532ce77199f0e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:47,991 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:47,993 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bd8975e71cd1de4115532ce77199f0e columnFamilyName b 2024-12-07T01:22:47,993 DEBUG [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:47,994 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(327): Store=8bd8975e71cd1de4115532ce77199f0e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:47,994 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:47,996 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bd8975e71cd1de4115532ce77199f0e columnFamilyName c 2024-12-07T01:22:47,996 DEBUG [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:47,997 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(327): Store=8bd8975e71cd1de4115532ce77199f0e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:47,997 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:47,998 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:47,999 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,000 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,000 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,001 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:48,003 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,006 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:48,006 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8bd8975e71cd1de4115532ce77199f0e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63767869, jitterRate=-0.04978470504283905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:48,008 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8bd8975e71cd1de4115532ce77199f0e: Writing region info on filesystem at 1733534567987Initializing all the Stores at 1733534567988 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534567988Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534567988Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534567988Cleaning up temporary data from old regions at 1733534568000 (+12 ms)Region opened successfully at 1733534568008 (+8 ms) 2024-12-07T01:22:48,008 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8bd8975e71cd1de4115532ce77199f0e, disabling compactions & flushes 2024-12-07T01:22:48,008 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. 2024-12-07T01:22:48,009 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. 2024-12-07T01:22:48,009 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. after waiting 0 ms 2024-12-07T01:22:48,009 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. 2024-12-07T01:22:48,009 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. 2024-12-07T01:22:48,009 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8bd8975e71cd1de4115532ce77199f0e: Waiting for close lock at 1733534568008Disabling compacts and flushes for region at 1733534568008Disabling writes for close at 1733534568009 (+1 ms)Writing region close event to WAL at 1733534568009Closed at 1733534568009 2024-12-07T01:22:48,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741878_1055 (size=93) 2024-12-07T01:22:48,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741878_1055 (size=93) 2024-12-07T01:22:48,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741878_1055 (size=93) 2024-12-07T01:22:48,016 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:22:48,016 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23444869:(num 1733534567948) 2024-12-07T01:22:48,016 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:48,018 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:48,038 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019, exclude list is [], retry=0 2024-12-07T01:22:48,041 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:48,042 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:48,043 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:48,045 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 2024-12-07T01:22:48,045 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:48,045 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8bd8975e71cd1de4115532ce77199f0e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:48,046 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:48,046 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,046 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,048 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,049 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bd8975e71cd1de4115532ce77199f0e columnFamilyName a 2024-12-07T01:22:48,050 DEBUG [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:48,050 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(327): Store=8bd8975e71cd1de4115532ce77199f0e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:48,050 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,051 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bd8975e71cd1de4115532ce77199f0e columnFamilyName b 2024-12-07T01:22:48,052 DEBUG [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:48,052 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(327): Store=8bd8975e71cd1de4115532ce77199f0e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:48,052 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,054 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bd8975e71cd1de4115532ce77199f0e columnFamilyName c 2024-12-07T01:22:48,054 DEBUG [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:48,054 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(327): Store=8bd8975e71cd1de4115532ce77199f0e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:48,054 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,055 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,056 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,058 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,058 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,058 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:48,060 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:48,061 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8bd8975e71cd1de4115532ce77199f0e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63523830, jitterRate=-0.05342116951942444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:48,061 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8bd8975e71cd1de4115532ce77199f0e: Writing region info on filesystem at 1733534568046Initializing all the Stores at 1733534568047 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534568047Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534568048 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534568048Cleaning up temporary data from old regions at 1733534568058 (+10 ms)Region opened successfully at 1733534568061 (+3 ms) 2024-12-07T01:22:48,066 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733534568065/Put/seqid=0 2024-12-07T01:22:48,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741881_1058 (size=4826) 2024-12-07T01:22:48,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741881_1058 (size=4826) 2024-12-07T01:22:48,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741881_1058 (size=4826) 2024-12-07T01:22:48,074 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36673/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 8bd8975e71cd1de4115532ce77199f0e/a 2024-12-07T01:22:48,083 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-07T01:22:48,083 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-07T01:22:48,083 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8bd8975e71cd1de4115532ce77199f0e: 2024-12-07T01:22:48,085 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/a/2fec8cf6515245fc811cbce37845796e_SeqId_3_ 2024-12-07T01:22:48,086 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36673/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 8bd8975e71cd1de4115532ce77199f0e/a as hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/a/2fec8cf6515245fc811cbce37845796e_SeqId_3_ - updating store file list. 2024-12-07T01:22:48,092 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2fec8cf6515245fc811cbce37845796e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-07T01:22:48,093 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/a/2fec8cf6515245fc811cbce37845796e_SeqId_3_ into 8bd8975e71cd1de4115532ce77199f0e/a 2024-12-07T01:22:48,093 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36673/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 8bd8975e71cd1de4115532ce77199f0e/a (new location: hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/a/2fec8cf6515245fc811cbce37845796e_SeqId_3_) 2024-12-07T01:22:48,129 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019, size=0 (0bytes) 2024-12-07T01:22:48,129 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 might be still open, length is 0 2024-12-07T01:22:48,130 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 2024-12-07T01:22:48,130 WARN [IPC Server handler 0 on default port 36673 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-07T01:22:48,131 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 after 1ms 2024-12-07T01:22:48,502 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:40930 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45293:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40930 dst: /127.0.0.1:45293 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45293 remote=/127.0.0.1:40930]. Total timeout mills is 60000, 59597 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:48,503 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:47106 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46343:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47106 dst: /127.0.0.1:46343 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:48,503 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:36888 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33463:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36888 dst: /127.0.0.1:33463 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:48,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741880_1059 (size=504) 2024-12-07T01:22:48,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741880_1059 (size=504) 2024-12-07T01:22:48,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741880_1059 (size=504) 2024-12-07T01:22:52,132 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 after 4002ms 2024-12-07T01:22:52,139 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 took 4010ms 2024-12-07T01:22:52,142 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019; continuing. 2024-12-07T01:22:52,142 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 so closing down 2024-12-07T01:22:52,143 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:52,146 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733534568019.temp 2024-12-07T01:22:52,148 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005-wal.1733534568019.temp 2024-12-07T01:22:52,148 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:52,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741882_1060 (size=238) 2024-12-07T01:22:52,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741882_1060 (size=238) 2024-12-07T01:22:52,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741882_1060 (size=238) 2024-12-07T01:22:52,156 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005-wal.1733534568019.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-07T01:22:52,158 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005-wal.1733534568019.temp to hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005 2024-12-07T01:22:52,158 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019, size=0, length=0, corrupted=false, cancelled=false 2024-12-07T01:22:52,158 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019, journal: Splitting hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019, size=0 (0bytes) at 1733534568129Finishing writing output for hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 so closing down at 1733534572142 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005-wal.1733534568019.temp at 1733534572148 (+6 ms)3 split writer threads finished at 1733534572148Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005-wal.1733534568019.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733534572157 (+9 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005-wal.1733534568019.temp to hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005 at 1733534572158 (+1 ms)Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019, size=0, length=0, corrupted=false, cancelled=false at 1733534572158 2024-12-07T01:22:52,160 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 to hdfs://localhost:36673/hbase/oldWALs/wal.1733534568019 2024-12-07T01:22:52,160 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005 2024-12-07T01:22:52,161 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:52,163 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:52,176 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534572163, exclude list is [], retry=0 2024-12-07T01:22:52,179 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:52,180 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:52,180 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:52,182 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534572163 2024-12-07T01:22:52,182 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991)] 2024-12-07T01:22:52,182 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8bd8975e71cd1de4115532ce77199f0e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:52,183 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:52,183 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,183 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,186 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,187 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bd8975e71cd1de4115532ce77199f0e columnFamilyName a 2024-12-07T01:22:52,187 DEBUG [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:52,195 DEBUG [StoreFileOpener-8bd8975e71cd1de4115532ce77199f0e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2fec8cf6515245fc811cbce37845796e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-07T01:22:52,195 DEBUG [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/a/2fec8cf6515245fc811cbce37845796e_SeqId_3_ 2024-12-07T01:22:52,195 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(327): Store=8bd8975e71cd1de4115532ce77199f0e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:52,195 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,196 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bd8975e71cd1de4115532ce77199f0e columnFamilyName b 2024-12-07T01:22:52,196 DEBUG [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:52,197 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(327): Store=8bd8975e71cd1de4115532ce77199f0e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:52,197 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,198 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8bd8975e71cd1de4115532ce77199f0e columnFamilyName c 2024-12-07T01:22:52,198 DEBUG [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:52,199 INFO [StoreOpener-8bd8975e71cd1de4115532ce77199f0e-1 {}] regionserver.HStore(327): Store=8bd8975e71cd1de4115532ce77199f0e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:52,199 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,200 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,202 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,203 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005 2024-12-07T01:22:52,226 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005 2024-12-07T01:22:52,226 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8bd8975e71cd1de4115532ce77199f0e 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-07T01:22:52,241 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/.tmp/a/9de8dc92249d4c36bd69bb57c21d1f33 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733534568097/Put/seqid=0 2024-12-07T01:22:52,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741884_1062 (size=5149) 2024-12-07T01:22:52,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741884_1062 (size=5149) 2024-12-07T01:22:52,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741884_1062 (size=5149) 2024-12-07T01:22:52,654 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/.tmp/a/9de8dc92249d4c36bd69bb57c21d1f33 2024-12-07T01:22:52,665 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/.tmp/a/9de8dc92249d4c36bd69bb57c21d1f33 as hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/a/9de8dc92249d4c36bd69bb57c21d1f33 2024-12-07T01:22:52,670 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/a/9de8dc92249d4c36bd69bb57c21d1f33, entries=1, sequenceid=5, filesize=5.0 K 2024-12-07T01:22:52,671 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 8bd8975e71cd1de4115532ce77199f0e in 445ms, sequenceid=5, compaction requested=false; wal=null 2024-12-07T01:22:52,671 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/0000000000000000005 2024-12-07T01:22:52,673 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,673 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,673 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:52,676 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8bd8975e71cd1de4115532ce77199f0e 2024-12-07T01:22:52,678 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8bd8975e71cd1de4115532ce77199f0e/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-07T01:22:52,679 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8bd8975e71cd1de4115532ce77199f0e; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61904873, jitterRate=-0.07754550874233246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:52,679 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8bd8975e71cd1de4115532ce77199f0e: Writing region info on filesystem at 1733534572183Initializing all the Stores at 1733534572184 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534572184Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534572186 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534572186Obtaining lock to block concurrent updates at 1733534572226 (+40 ms)Preparing flush snapshotting stores in 8bd8975e71cd1de4115532ce77199f0e at 1733534572226Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733534572226Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. at 1733534572226Flushing 8bd8975e71cd1de4115532ce77199f0e/a: creating writer at 1733534572227 (+1 ms)Flushing 8bd8975e71cd1de4115532ce77199f0e/a: appending metadata at 1733534572240 (+13 ms)Flushing 8bd8975e71cd1de4115532ce77199f0e/a: closing flushed file at 1733534572241 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@533d057d: reopening flushed file at 1733534572664 (+423 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 8bd8975e71cd1de4115532ce77199f0e in 445ms, sequenceid=5, compaction requested=false; wal=null at 1733534572671 (+7 ms)Cleaning up temporary data from old regions at 1733534572673 (+2 ms)Region opened successfully at 1733534572679 (+6 ms) 2024-12-07T01:22:52,683 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8bd8975e71cd1de4115532ce77199f0e, disabling compactions & flushes 2024-12-07T01:22:52,684 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. 2024-12-07T01:22:52,684 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. 2024-12-07T01:22:52,684 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. after waiting 0 ms 2024-12-07T01:22:52,684 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. 2024-12-07T01:22:52,685 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733534567921.8bd8975e71cd1de4115532ce77199f0e. 2024-12-07T01:22:52,685 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8bd8975e71cd1de4115532ce77199f0e: Waiting for close lock at 1733534572683Disabling compacts and flushes for region at 1733534572683Disabling writes for close at 1733534572684 (+1 ms)Writing region close event to WAL at 1733534572684Closed at 1733534572684 2024-12-07T01:22:52,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741883_1061 (size=93) 2024-12-07T01:22:52,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741883_1061 (size=93) 2024-12-07T01:22:52,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741883_1061 (size=93) 2024-12-07T01:22:52,690 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:22:52,690 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733534572163) 2024-12-07T01:22:52,707 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=414 (was 411) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:36673 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1197420602_22 at /127.0.0.1:58414 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1197420602_22 at /127.0.0.1:42354 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:36673 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1197420602_22 at /127.0.0.1:60256 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=4934 (was 5032) 2024-12-07T01:22:52,720 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=414, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=4933 2024-12-07T01:22:52,736 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:52,741 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T01:22:52,744 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ec1863dc21e5,45579,1733534557376 2024-12-07T01:22:52,746 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6e021000 2024-12-07T01:22:52,747 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T01:22:52,749 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T01:22:52,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T01:22:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-07T01:22:52,761 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T01:22:52,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-07T01:22:52,763 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:52,765 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T01:22:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T01:22:52,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741885_1063 (size=694) 2024-12-07T01:22:52,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741885_1063 (size=694) 2024-12-07T01:22:52,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741885_1063 (size=694) 2024-12-07T01:22:52,777 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3848f991c723c9b72b68e9365062aeee, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe 2024-12-07T01:22:52,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741886_1064 (size=77) 2024-12-07T01:22:52,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741886_1064 (size=77) 2024-12-07T01:22:52,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741886_1064 (size=77) 2024-12-07T01:22:52,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:52,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 3848f991c723c9b72b68e9365062aeee, disabling compactions & flushes 2024-12-07T01:22:52,786 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:52,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:52,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. after waiting 0 ms 2024-12-07T01:22:52,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:52,786 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:52,786 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3848f991c723c9b72b68e9365062aeee: Waiting for close lock at 1733534572786Disabling compacts and flushes for region at 1733534572786Disabling writes for close at 1733534572786Writing region close event to WAL at 1733534572786Closed at 1733534572786 2024-12-07T01:22:52,788 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T01:22:52,794 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733534572788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733534572788"}]},"ts":"1733534572788"} 2024-12-07T01:22:52,797 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T01:22:52,798 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T01:22:52,801 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733534572799"}]},"ts":"1733534572799"} 2024-12-07T01:22:52,805 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-07T01:22:52,805 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ec1863dc21e5=0} racks are {/default-rack=0} 2024-12-07T01:22:52,807 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T01:22:52,807 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T01:22:52,807 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-07T01:22:52,807 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T01:22:52,807 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T01:22:52,807 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-07T01:22:52,807 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T01:22:52,807 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T01:22:52,807 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-07T01:22:52,807 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T01:22:52,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, ASSIGN}] 2024-12-07T01:22:52,810 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, ASSIGN 2024-12-07T01:22:52,811 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, ASSIGN; state=OFFLINE, location=ec1863dc21e5,38575,1733534558286; forceNewPlan=false, retain=false 2024-12-07T01:22:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T01:22:52,965 INFO [ec1863dc21e5:45579 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T01:22:52,967 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=OPENING, regionLocation=ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:52,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, ASSIGN because future has completed 2024-12-07T01:22:52,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,38575,1733534558286}] 2024-12-07T01:22:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T01:22:53,128 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T01:22:53,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46487, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T01:22:53,135 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,135 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3848f991c723c9b72b68e9365062aeee, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:53,135 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,136 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:53,136 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,136 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,138 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,139 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf1 2024-12-07T01:22:53,140 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:53,140 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:53,140 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,142 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf2 2024-12-07T01:22:53,142 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:53,142 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:53,143 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,143 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,144 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,145 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,145 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,146 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-07T01:22:53,147 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,152 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:53,153 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3848f991c723c9b72b68e9365062aeee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60104184, jitterRate=-0.1043778657913208}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-07T01:22:53,153 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,154 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3848f991c723c9b72b68e9365062aeee: Running coprocessor pre-open hook at 1733534573136Writing region info on filesystem at 1733534573136Initializing all the Stores at 1733534573137 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534573137Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534573137Cleaning up temporary data from old regions at 1733534573145 (+8 ms)Running coprocessor post-open hooks at 1733534573153 (+8 ms)Region opened successfully at 1733534573154 (+1 ms) 2024-12-07T01:22:53,156 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., pid=6, masterSystemTime=1733534573128 2024-12-07T01:22:53,158 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,159 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,160 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=OPEN, openSeqNum=2, regionLocation=ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:53,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,38575,1733534558286 because future has completed 2024-12-07T01:22:53,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T01:22:53,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,38575,1733534558286 in 191 msec 2024-12-07T01:22:53,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T01:22:53,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, ASSIGN in 360 msec 2024-12-07T01:22:53,173 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T01:22:53,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733534573173"}]},"ts":"1733534573173"} 2024-12-07T01:22:53,177 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-07T01:22:53,178 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T01:22:53,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 424 msec 2024-12-07T01:22:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T01:22:53,400 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-07T01:22:53,400 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-07T01:22:53,402 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T01:22:53,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-07T01:22:53,410 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T01:22:53,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-07T01:22:53,424 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=2] 2024-12-07T01:22:53,426 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T01:22:53,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T01:22:53,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=3848f991c723c9b72b68e9365062aeee, source=ec1863dc21e5,38575,1733534558286, destination=ec1863dc21e5,35781,1733534558145, warming up region on ec1863dc21e5,35781,1733534558145 2024-12-07T01:22:53,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T01:22:53,447 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=3848f991c723c9b72b68e9365062aeee, source=ec1863dc21e5,38575,1733534558286, destination=ec1863dc21e5,35781,1733534558145, running balancer 2024-12-07T01:22:53,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37577, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T01:22:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE 2024-12-07T01:22:53,449 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE 2024-12-07T01:22:53,452 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=CLOSING, regionLocation=ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:53,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3848f991c723c9b72b68e9365062aeee, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:53,455 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE because future has completed 2024-12-07T01:22:53,456 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T01:22:53,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,38575,1733534558286}] 2024-12-07T01:22:53,456 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf1 2024-12-07T01:22:53,456 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:53,457 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:53,457 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,458 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf2 2024-12-07T01:22:53,458 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:53,459 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] regionserver.HRegion(1722): Closing 3848f991c723c9b72b68e9365062aeee, disabling compactions & flushes 2024-12-07T01:22:53,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. after waiting 0 ms 2024-12-07T01:22:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] regionserver.HRegion(1676): Region close journal for 3848f991c723c9b72b68e9365062aeee: Waiting for close lock at 1733534573459Disabling compacts and flushes for region at 1733534573459Disabling writes for close at 1733534573459Writing region close event to WAL at 1733534573460 (+1 ms)Closed at 1733534573460 2024-12-07T01:22:53,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-07T01:22:53,620 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,621 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T01:22:53,622 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 3848f991c723c9b72b68e9365062aeee, disabling compactions & flushes 2024-12-07T01:22:53,622 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,622 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,622 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. after waiting 0 ms 2024-12-07T01:22:53,622 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,623 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 3848f991c723c9b72b68e9365062aeee 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-07T01:22:53,642 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf1/78d0a2d8b33c492e9ac996ad63cb15c9 is 35, key is r1/cf1:q/1733534573429/Put/seqid=0 2024-12-07T01:22:53,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741887_1065 (size=4783) 2024-12-07T01:22:53,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741887_1065 (size=4783) 2024-12-07T01:22:53,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741887_1065 (size=4783) 2024-12-07T01:22:53,649 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf1/78d0a2d8b33c492e9ac996ad63cb15c9 2024-12-07T01:22:53,656 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf1/78d0a2d8b33c492e9ac996ad63cb15c9 as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/78d0a2d8b33c492e9ac996ad63cb15c9 2024-12-07T01:22:53,662 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/78d0a2d8b33c492e9ac996ad63cb15c9, entries=1, sequenceid=5, filesize=4.7 K 2024-12-07T01:22:53,664 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 3848f991c723c9b72b68e9365062aeee in 41ms, sequenceid=5, compaction requested=false 2024-12-07T01:22:53,664 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-07T01:22:53,668 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-07T01:22:53,671 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:53,671 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 3848f991c723c9b72b68e9365062aeee: Waiting for close lock at 1733534573622Running coprocessor pre-close hooks at 1733534573622Disabling compacts and flushes for region at 1733534573622Disabling writes for close at 1733534573622Obtaining lock to block concurrent updates at 1733534573623 (+1 ms)Preparing flush snapshotting stores in 3848f991c723c9b72b68e9365062aeee at 1733534573623Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733534573623Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. at 1733534573624 (+1 ms)Flushing 3848f991c723c9b72b68e9365062aeee/cf1: creating writer at 1733534573624Flushing 3848f991c723c9b72b68e9365062aeee/cf1: appending metadata at 1733534573642 (+18 ms)Flushing 3848f991c723c9b72b68e9365062aeee/cf1: closing flushed file at 1733534573642Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@271a8d2b: reopening flushed file at 1733534573655 (+13 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 3848f991c723c9b72b68e9365062aeee in 41ms, sequenceid=5, compaction requested=false at 1733534573664 (+9 ms)Writing region close event to WAL at 1733534573665 (+1 ms)Running coprocessor post-close hooks at 1733534573669 (+4 ms)Closed at 1733534573671 (+2 ms) 2024-12-07T01:22:53,671 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 3848f991c723c9b72b68e9365062aeee move to ec1863dc21e5,35781,1733534558145 record at close sequenceid=5 2024-12-07T01:22:53,674 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:53,675 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=CLOSED 2024-12-07T01:22:53,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,38575,1733534558286 because future has completed 2024-12-07T01:22:53,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T01:22:53,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,38575,1733534558286 in 223 msec 2024-12-07T01:22:53,683 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE; state=CLOSED, location=ec1863dc21e5,35781,1733534558145; forceNewPlan=false, retain=false 2024-12-07T01:22:53,833 INFO [ec1863dc21e5:45579 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T01:22:53,834 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=OPENING, regionLocation=ec1863dc21e5,35781,1733534558145 2024-12-07T01:22:53,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE because future has completed 2024-12-07T01:22:53,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,35781,1733534558145}] 2024-12-07T01:22:54,004 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,004 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3848f991c723c9b72b68e9365062aeee, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:54,004 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,004 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:54,004 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,005 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,006 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,007 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf1 2024-12-07T01:22:54,007 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:54,016 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/78d0a2d8b33c492e9ac996ad63cb15c9 2024-12-07T01:22:54,016 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:54,016 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,017 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf2 2024-12-07T01:22:54,017 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:54,018 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:54,018 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,019 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,021 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,022 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,022 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,023 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-07T01:22:54,025 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,026 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 3848f991c723c9b72b68e9365062aeee; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64157741, jitterRate=-0.04397515952587128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-07T01:22:54,026 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,027 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3848f991c723c9b72b68e9365062aeee: Running coprocessor pre-open hook at 1733534574005Writing region info on filesystem at 1733534574005Initializing all the Stores at 1733534574006 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534574006Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534574006Cleaning up temporary data from old regions at 1733534574022 (+16 ms)Running coprocessor post-open hooks at 1733534574026 (+4 ms)Region opened successfully at 1733534574027 (+1 ms) 2024-12-07T01:22:54,029 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., pid=9, masterSystemTime=1733534573999 2024-12-07T01:22:54,031 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,031 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=OPEN, openSeqNum=9, regionLocation=ec1863dc21e5,35781,1733534558145 2024-12-07T01:22:54,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,35781,1733534558145 because future has completed 2024-12-07T01:22:54,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-07T01:22:54,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,35781,1733534558145 in 190 msec 2024-12-07T01:22:54,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE in 592 msec 2024-12-07T01:22:54,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T01:22:54,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T01:22:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:37256 deadline: 1733534634063, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec1863dc21e5 port=35781 startCode=1733534558145. As of locationSeqNum=5. 2024-12-07T01:22:54,086 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec1863dc21e5 port=35781 startCode=1733534558145. As of locationSeqNum=5. 2024-12-07T01:22:54,087 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec1863dc21e5 port=35781 startCode=1733534558145. As of locationSeqNum=5. 2024-12-07T01:22:54,087 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,35781,1733534558145, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec1863dc21e5 port=35781 startCode=1733534558145. As of locationSeqNum=5. 2024-12-07T01:22:54,198 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T01:22:54,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T01:22:54,214 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3848f991c723c9b72b68e9365062aeee 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-07T01:22:54,229 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf1/c39c7569878e4c7fac30ae10de9669a1 is 29, key is r1/cf1:/1733534574204/DeleteFamily/seqid=0 2024-12-07T01:22:54,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741888_1066 (size=4906) 2024-12-07T01:22:54,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741888_1066 (size=4906) 2024-12-07T01:22:54,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741888_1066 (size=4906) 2024-12-07T01:22:54,236 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf1/c39c7569878e4c7fac30ae10de9669a1 2024-12-07T01:22:54,242 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c39c7569878e4c7fac30ae10de9669a1 2024-12-07T01:22:54,256 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf2/8a513d357a214e8188798a3d5437cb32 is 29, key is r1/cf2:/1733534574204/DeleteFamily/seqid=0 2024-12-07T01:22:54,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741889_1067 (size=4906) 2024-12-07T01:22:54,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741889_1067 (size=4906) 2024-12-07T01:22:54,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741889_1067 (size=4906) 2024-12-07T01:22:54,264 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf2/8a513d357a214e8188798a3d5437cb32 2024-12-07T01:22:54,269 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8a513d357a214e8188798a3d5437cb32 2024-12-07T01:22:54,270 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf1/c39c7569878e4c7fac30ae10de9669a1 as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/c39c7569878e4c7fac30ae10de9669a1 2024-12-07T01:22:54,277 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c39c7569878e4c7fac30ae10de9669a1 2024-12-07T01:22:54,277 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/c39c7569878e4c7fac30ae10de9669a1, entries=1, sequenceid=12, filesize=4.8 K 2024-12-07T01:22:54,278 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf2/8a513d357a214e8188798a3d5437cb32 as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/8a513d357a214e8188798a3d5437cb32 2024-12-07T01:22:54,283 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8a513d357a214e8188798a3d5437cb32 2024-12-07T01:22:54,284 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/8a513d357a214e8188798a3d5437cb32, entries=1, sequenceid=12, filesize=4.8 K 2024-12-07T01:22:54,285 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 3848f991c723c9b72b68e9365062aeee in 71ms, sequenceid=12, compaction requested=false 2024-12-07T01:22:54,285 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3848f991c723c9b72b68e9365062aeee: 2024-12-07T01:22:54,287 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-07T01:22:54,289 DEBUG [Time-limited test {}] regionserver.HStore(1541): 3848f991c723c9b72b68e9365062aeee/cf1 is initiating major compaction (all files) 2024-12-07T01:22:54,289 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T01:22:54,289 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:54,289 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 3848f991c723c9b72b68e9365062aeee/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,290 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/78d0a2d8b33c492e9ac996ad63cb15c9, hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/c39c7569878e4c7fac30ae10de9669a1] into tmpdir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp, totalSize=9.5 K 2024-12-07T01:22:54,291 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 78d0a2d8b33c492e9ac996ad63cb15c9, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733534573429 2024-12-07T01:22:54,291 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c39c7569878e4c7fac30ae10de9669a1, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-07T01:22:54,301 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 3848f991c723c9b72b68e9365062aeee#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T01:22:54,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741890_1068 (size=4626) 2024-12-07T01:22:54,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741890_1068 (size=4626) 2024-12-07T01:22:54,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741890_1068 (size=4626) 2024-12-07T01:22:54,315 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf1/b235584b180b435396335a00eb42332d as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/b235584b180b435396335a00eb42332d 2024-12-07T01:22:54,328 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 3848f991c723c9b72b68e9365062aeee/cf1 of 3848f991c723c9b72b68e9365062aeee into b235584b180b435396335a00eb42332d(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T01:22:54,329 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 3848f991c723c9b72b68e9365062aeee: 2024-12-07T01:22:54,329 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-07T01:22:54,329 DEBUG [Time-limited test {}] regionserver.HStore(1541): 3848f991c723c9b72b68e9365062aeee/cf2 is initiating major compaction (all files) 2024-12-07T01:22:54,329 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T01:22:54,329 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T01:22:54,329 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 3848f991c723c9b72b68e9365062aeee/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,329 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/8a513d357a214e8188798a3d5437cb32] into tmpdir=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp, totalSize=4.8 K 2024-12-07T01:22:54,330 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8a513d357a214e8188798a3d5437cb32, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-07T01:22:54,336 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 3848f991c723c9b72b68e9365062aeee#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T01:22:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741891_1069 (size=4592) 2024-12-07T01:22:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741891_1069 (size=4592) 2024-12-07T01:22:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741891_1069 (size=4592) 2024-12-07T01:22:54,350 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/.tmp/cf2/ff249bc8ceef4fd299c452a0029cc198 as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/ff249bc8ceef4fd299c452a0029cc198 2024-12-07T01:22:54,358 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 3848f991c723c9b72b68e9365062aeee/cf2 of 3848f991c723c9b72b68e9365062aeee into ff249bc8ceef4fd299c452a0029cc198(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T01:22:54,358 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 3848f991c723c9b72b68e9365062aeee: 2024-12-07T01:22:54,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=3848f991c723c9b72b68e9365062aeee, source=ec1863dc21e5,35781,1733534558145, destination=ec1863dc21e5,38575,1733534558286, warming up region on ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:54,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=3848f991c723c9b72b68e9365062aeee, source=ec1863dc21e5,35781,1733534558145, destination=ec1863dc21e5,38575,1733534558286, running balancer 2024-12-07T01:22:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE 2024-12-07T01:22:54,364 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE 2024-12-07T01:22:54,366 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=CLOSING, regionLocation=ec1863dc21e5,35781,1733534558145 2024-12-07T01:22:54,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3848f991c723c9b72b68e9365062aeee, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:54,367 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,368 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf1 2024-12-07T01:22:54,368 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:54,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE because future has completed 2024-12-07T01:22:54,370 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T01:22:54,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,35781,1733534558145}] 2024-12-07T01:22:54,377 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/78d0a2d8b33c492e9ac996ad63cb15c9 2024-12-07T01:22:54,381 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/b235584b180b435396335a00eb42332d 2024-12-07T01:22:54,386 INFO [StoreFileOpener-3848f991c723c9b72b68e9365062aeee-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c39c7569878e4c7fac30ae10de9669a1 2024-12-07T01:22:54,386 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/c39c7569878e4c7fac30ae10de9669a1 2024-12-07T01:22:54,387 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:54,387 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,388 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf2 2024-12-07T01:22:54,388 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:54,394 INFO [StoreFileOpener-3848f991c723c9b72b68e9365062aeee-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8a513d357a214e8188798a3d5437cb32 2024-12-07T01:22:54,395 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/8a513d357a214e8188798a3d5437cb32 2024-12-07T01:22:54,400 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/ff249bc8ceef4fd299c452a0029cc198 2024-12-07T01:22:54,401 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] regionserver.HRegion(1722): Closing 3848f991c723c9b72b68e9365062aeee, disabling compactions & flushes 2024-12-07T01:22:54,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. after waiting 0 ms 2024-12-07T01:22:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38575 {}] regionserver.HRegion(1676): Region close journal for 3848f991c723c9b72b68e9365062aeee: Waiting for close lock at 1733534574401Disabling compacts and flushes for region at 1733534574401Disabling writes for close at 1733534574401Writing region close event to WAL at 1733534574402 (+1 ms)Closed at 1733534574402 2024-12-07T01:22:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-07T01:22:54,524 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,524 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T01:22:54,525 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 3848f991c723c9b72b68e9365062aeee, disabling compactions & flushes 2024-12-07T01:22:54,525 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,525 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,525 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. after waiting 0 ms 2024-12-07T01:22:54,525 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,525 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/78d0a2d8b33c492e9ac996ad63cb15c9, hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/c39c7569878e4c7fac30ae10de9669a1] to archive 2024-12-07T01:22:54,529 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T01:22:54,535 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/78d0a2d8b33c492e9ac996ad63cb15c9 to hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/78d0a2d8b33c492e9ac996ad63cb15c9 2024-12-07T01:22:54,538 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/c39c7569878e4c7fac30ae10de9669a1 to hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/c39c7569878e4c7fac30ae10de9669a1 2024-12-07T01:22:54,550 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/8a513d357a214e8188798a3d5437cb32] to archive 2024-12-07T01:22:54,551 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T01:22:54,553 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/8a513d357a214e8188798a3d5437cb32 to hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/8a513d357a214e8188798a3d5437cb32 2024-12-07T01:22:54,558 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-07T01:22:54,559 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,559 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 3848f991c723c9b72b68e9365062aeee: Waiting for close lock at 1733534574524Running coprocessor pre-close hooks at 1733534574524Disabling compacts and flushes for region at 1733534574524Disabling writes for close at 1733534574525 (+1 ms)Writing region close event to WAL at 1733534574555 (+30 ms)Running coprocessor post-close hooks at 1733534574559 (+4 ms)Closed at 1733534574559 2024-12-07T01:22:54,559 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 3848f991c723c9b72b68e9365062aeee move to ec1863dc21e5,38575,1733534558286 record at close sequenceid=12 2024-12-07T01:22:54,562 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,563 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=CLOSED 2024-12-07T01:22:54,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,35781,1733534558145 because future has completed 2024-12-07T01:22:54,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-07T01:22:54,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,35781,1733534558145 in 196 msec 2024-12-07T01:22:54,570 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE; state=CLOSED, location=ec1863dc21e5,38575,1733534558286; forceNewPlan=false, retain=false 2024-12-07T01:22:54,721 INFO [ec1863dc21e5:45579 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T01:22:54,721 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=OPENING, regionLocation=ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:54,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE because future has completed 2024-12-07T01:22:54,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,38575,1733534558286}] 2024-12-07T01:22:54,884 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,885 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 3848f991c723c9b72b68e9365062aeee, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:54,885 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,885 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:54,885 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,885 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,887 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,888 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf1 2024-12-07T01:22:54,888 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:54,895 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/b235584b180b435396335a00eb42332d 2024-12-07T01:22:54,895 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:54,895 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,897 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf2 2024-12-07T01:22:54,897 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:54,905 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/ff249bc8ceef4fd299c452a0029cc198 2024-12-07T01:22:54,905 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:54,905 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,906 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,907 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,908 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,908 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,909 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-07T01:22:54,910 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,911 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 3848f991c723c9b72b68e9365062aeee; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71472720, jitterRate=0.06502652168273926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-07T01:22:54,911 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,912 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 3848f991c723c9b72b68e9365062aeee: Running coprocessor pre-open hook at 1733534574885Writing region info on filesystem at 1733534574885Initializing all the Stores at 1733534574886 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534574886Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534574887 (+1 ms)Cleaning up temporary data from old regions at 1733534574908 (+21 ms)Running coprocessor post-open hooks at 1733534574911 (+3 ms)Region opened successfully at 1733534574912 (+1 ms) 2024-12-07T01:22:54,913 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., pid=12, masterSystemTime=1733534574881 2024-12-07T01:22:54,915 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,915 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,916 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=OPEN, openSeqNum=18, regionLocation=ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:54,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,38575,1733534558286 because future has completed 2024-12-07T01:22:54,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-07T01:22:54,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,38575,1733534558286 in 194 msec 2024-12-07T01:22:54,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, REOPEN/MOVE in 560 msec 2024-12-07T01:22:54,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T01:22:54,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T01:22:54,970 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server ec1863dc21e5,38575,1733534558286: testing ***** 2024-12-07T01:22:54,970 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-07T01:22:54,972 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-07T01:22:54,973 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-07T01:22:54,976 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-07T01:22:54,977 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-07T01:22:54,983 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 276468760 }, "NonHeapMemoryUsage": { "committed": 170065920, "init": 7667712, "max": -1, "used": 167552736 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ec1863dc21e5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ec1863dc21e5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:61145", "tag.serverName": "ec1863dc21e5,35853,1733534558237", "tag.clusterId": "39d45156-d657-4acd-921b-b0e3165a848f", "tag.Context": "regionserver", "tag.Hostname": "ec1863dc21e5", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733534558237, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 3, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 8, "Put_mean": 5, "Put_25th_percentile": 3, "Put_median": 5, "Put_75th_percentile": 6, "Put_90th_percentile": 7, "Put_95th_percentile": 7, "Put_98th_percentile": 7, "Put_99th_percentile": 7, "Put_99.9th_percentile": 7, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-07T01:22:54,986 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45579 {}] master.MasterRpcServices(700): ec1863dc21e5,38575,1733534558286 reported a fatal error: ***** ABORTING region server ec1863dc21e5,38575,1733534558286: testing ***** 2024-12-07T01:22:54,988 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec1863dc21e5,38575,1733534558286' ***** 2024-12-07T01:22:54,988 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-07T01:22:54,988 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T01:22:54,989 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T01:22:54,989 INFO [RS:2;ec1863dc21e5:38575 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-07T01:22:54,989 INFO [RS:2;ec1863dc21e5:38575 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-07T01:22:54,989 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(3091): Received CLOSE for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,989 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(956): aborting server ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:54,989 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T01:22:54,990 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3848f991c723c9b72b68e9365062aeee, disabling compactions & flushes 2024-12-07T01:22:54,990 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,990 INFO [RS:2;ec1863dc21e5:38575 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ec1863dc21e5:38575. 2024-12-07T01:22:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:47300 deadline: 1733534634989, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec1863dc21e5 port=38575 startCode=1733534558286. As of locationSeqNum=12. 2024-12-07T01:22:54,990 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,990 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. after waiting 0 ms 2024-12-07T01:22:54,990 DEBUG [RS:2;ec1863dc21e5:38575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T01:22:54,990 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,990 DEBUG [RS:2;ec1863dc21e5:38575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:22:54,990 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,35781,1733534558145, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,35781,1733534558145, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec1863dc21e5 port=38575 startCode=1733534558286. As of locationSeqNum=12. 2024-12-07T01:22:54,991 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,35781,1733534558145, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec1863dc21e5 port=38575 startCode=1733534558286. As of locationSeqNum=12. 2024-12-07T01:22:54,991 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-07T01:22:54,991 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,35781,1733534558145, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec1863dc21e5 port=38575 startCode=1733534558286. As of locationSeqNum=12. 2024-12-07T01:22:54,991 DEBUG [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(1325): Online Regions={3848f991c723c9b72b68e9365062aeee=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.} 2024-12-07T01:22:54,991 DEBUG [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(1351): Waiting on 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:54,992 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:54,992 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3848f991c723c9b72b68e9365062aeee: Waiting for close lock at 1733534574989Running coprocessor pre-close hooks at 1733534574990 (+1 ms)Disabling compacts and flushes for region at 1733534574990Disabling writes for close at 1733534574990Writing region close event to WAL at 1733534574991 (+1 ms)Running coprocessor post-close hooks at 1733534574991Closed at 1733534574992 (+1 ms) 2024-12-07T01:22:54,992 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:55,027 INFO [regionserver/ec1863dc21e5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:22:55,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ec1863dc21e5,38575,1733534558286 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:55,102 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ec1863dc21e5,38575,1733534558286 aborting 2024-12-07T01:22:55,102 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ec1863dc21e5,38575,1733534558286 aborting 2024-12-07T01:22:55,103 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=12 from cache 2024-12-07T01:22:55,191 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(976): stopping server ec1863dc21e5,38575,1733534558286; all regions closed. 2024-12-07T01:22:55,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741834_1010 (size=2142) 2024-12-07T01:22:55,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741834_1010 (size=2142) 2024-12-07T01:22:55,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741834_1010 (size=2142) 2024-12-07T01:22:55,196 DEBUG [RS:2;ec1863dc21e5:38575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:22:55,196 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:22:55,196 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T01:22:55,197 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.ChoreService(370): Chore service for: regionserver/ec1863dc21e5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-07T01:22:55,197 INFO [regionserver/ec1863dc21e5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T01:22:55,197 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T01:22:55,197 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T01:22:55,197 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T01:22:55,197 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T01:22:55,198 INFO [RS:2;ec1863dc21e5:38575 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38575 2024-12-07T01:22:55,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T01:22:55,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec1863dc21e5,38575,1733534558286 2024-12-07T01:22:55,273 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T01:22:55,276 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec1863dc21e5,38575,1733534558286] 2024-12-07T01:22:55,293 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec1863dc21e5,38575,1733534558286 already deleted, retry=false 2024-12-07T01:22:55,293 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ec1863dc21e5,38575,1733534558286 on ec1863dc21e5,45579,1733534557376 2024-12-07T01:22:55,300 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ec1863dc21e5,38575,1733534558286, splitWal=true, meta=false 2024-12-07T01:22:55,303 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for ec1863dc21e5,38575,1733534558286 (carryingMeta=false) ec1863dc21e5,38575,1733534558286/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@50624b77[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-07T01:22:55,303 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ec1863dc21e5,38575,1733534558286, splitWal=true, meta=false 2024-12-07T01:22:55,305 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): ec1863dc21e5,38575,1733534558286 had 1 regions 2024-12-07T01:22:55,306 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ec1863dc21e5,38575,1733534558286, splitWal=true, meta=false, isMeta: false 2024-12-07T01:22:55,307 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting 2024-12-07T01:22:55,309 INFO [PEWorker-2 {}] master.SplitWALManager(105): ec1863dc21e5,38575,1733534558286 WAL count=1, meta=false 2024-12-07T01:22:55,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ec1863dc21e5%2C38575%2C1733534558286.1733534559752}] 2024-12-07T01:22:55,311 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=18] 2024-12-07T01:22:55,312 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ec1863dc21e5:38575 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ec1863dc21e5/172.17.0.3:38575 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:55,313 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=18, error=java.net.ConnectException: Call to address=ec1863dc21e5:38575 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ec1863dc21e5/172.17.0.3:38575 2024-12-07T01:22:55,313 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-07T01:22:55,313 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=18 from cache 2024-12-07T01:22:55,313 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address ec1863dc21e5:38575 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ec1863dc21e5/172.17.0.3:38575 2024-12-07T01:22:55,316 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:55,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C38575%2C1733534558286.1733534559752, worker=ec1863dc21e5,35853,1733534558237}] 2024-12-07T01:22:55,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:22:55,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38575-0x101ad621d6e0003, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:22:55,383 INFO [RS:2;ec1863dc21e5:38575 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T01:22:55,383 INFO [RS:2;ec1863dc21e5:38575 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec1863dc21e5,38575,1733534558286; zookeeper connection closed. 2024-12-07T01:22:55,384 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4d01d2c2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4d01d2c2 2024-12-07T01:22:55,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35853 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-07T01:22:55,506 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752, size=2.1 K (2142bytes) 2024-12-07T01:22:55,506 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752 2024-12-07T01:22:55,507 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752 after 1ms 2024-12-07T01:22:55,511 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752 took 5ms 2024-12-07T01:22:55,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 3848f991c723c9b72b68e9365062aeee: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-07T01:22:55,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752 so closing down 2024-12-07T01:22:55,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:55,518 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:55,518 INFO [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-12-07T01:22:55,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752, journal: Splitting hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752, size=2.1 K (2142bytes) at 1733534575506Finishing writing output for hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752 so closing down at 1733534575518 (+12 ms)3 split writer threads finished at 1733534575518Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1733534575518 2024-12-07T01:22:55,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752 2024-12-07T01:22:55,519 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec1863dc21e5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-07T01:22:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45579 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-07T01:22:55,525 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting/ec1863dc21e5%2C38575%2C1733534558286.1733534559752 to hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/oldWALs 2024-12-07T01:22:55,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-07T01:22:55,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec1863dc21e5%2C38575%2C1733534558286.1733534559752, worker=ec1863dc21e5,35853,1733534558237 in 206 msec 2024-12-07T01:22:55,529 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:55,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-07T01:22:55,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure ec1863dc21e5%2C38575%2C1733534558286.1733534559752, worker=ec1863dc21e5,35853,1733534558237 in 219 msec 2024-12-07T01:22:55,536 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting dir is empty, no logs to split. 2024-12-07T01:22:55,536 INFO [PEWorker-2 {}] master.SplitWALManager(105): ec1863dc21e5,38575,1733534558286 WAL count=0, meta=false 2024-12-07T01:22:55,536 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if ec1863dc21e5,38575,1733534558286 WAL splitting is done? wals=0, meta=false 2024-12-07T01:22:55,538 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for ec1863dc21e5,38575,1733534558286 failed, ignore...File hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/WALs/ec1863dc21e5,38575,1733534558286-splitting does not exist. 2024-12-07T01:22:55,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, ASSIGN}] 2024-12-07T01:22:55,541 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, ASSIGN 2024-12-07T01:22:55,542 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T01:22:55,621 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=18] 2024-12-07T01:22:55,621 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to ec1863dc21e5:38575 this server is in the failed servers list 2024-12-07T01:22:55,622 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ec1863dc21e5:38575 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ec1863dc21e5:38575 2024-12-07T01:22:55,622 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ec1863dc21e5:38575 2024-12-07T01:22:55,622 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,38575,1733534558286, seqNum=18 from cache 2024-12-07T01:22:55,692 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(204): Hosts are {ec1863dc21e5=0} racks are {/default-rack=0} 2024-12-07T01:22:55,693 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T01:22:55,693 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T01:22:55,693 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T01:22:55,693 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T01:22:55,693 INFO [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T01:22:55,693 INFO [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T01:22:55,693 DEBUG [ec1863dc21e5:45579 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T01:22:55,693 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=OPENING, regionLocation=ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:55,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, ASSIGN because future has completed 2024-12-07T01:22:55,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,35853,1733534558237}] 2024-12-07T01:22:55,854 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:55,855 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 3848f991c723c9b72b68e9365062aeee, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:55,855 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,856 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:55,856 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,856 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,858 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,860 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf1 2024-12-07T01:22:55,860 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:55,867 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf1/b235584b180b435396335a00eb42332d 2024-12-07T01:22:55,867 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:55,867 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,868 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3848f991c723c9b72b68e9365062aeee columnFamilyName cf2 2024-12-07T01:22:55,868 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:55,875 DEBUG [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/cf2/ff249bc8ceef4fd299c452a0029cc198 2024-12-07T01:22:55,875 INFO [StoreOpener-3848f991c723c9b72b68e9365062aeee-1 {}] regionserver.HStore(327): Store=3848f991c723c9b72b68e9365062aeee/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:55,875 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,876 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,877 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,878 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,878 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,878 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-07T01:22:55,880 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,881 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 3848f991c723c9b72b68e9365062aeee; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73098092, jitterRate=0.08924645185470581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-07T01:22:55,881 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:22:55,882 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 3848f991c723c9b72b68e9365062aeee: Running coprocessor pre-open hook at 1733534575856Writing region info on filesystem at 1733534575856Initializing all the Stores at 1733534575858 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534575858Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534575858Cleaning up temporary data from old regions at 1733534575878 (+20 ms)Running coprocessor post-open hooks at 1733534575881 (+3 ms)Region opened successfully at 1733534575882 (+1 ms) 2024-12-07T01:22:55,883 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., pid=17, masterSystemTime=1733534575848 2024-12-07T01:22:55,885 DEBUG [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:55,885 INFO [RS_OPEN_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:22:55,886 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3848f991c723c9b72b68e9365062aeee, regionState=OPEN, openSeqNum=18, regionLocation=ec1863dc21e5,35853,1733534558237 2024-12-07T01:22:55,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,35853,1733534558237 because future has completed 2024-12-07T01:22:55,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-07T01:22:55,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 3848f991c723c9b72b68e9365062aeee, server=ec1863dc21e5,35853,1733534558237 in 193 msec 2024-12-07T01:22:55,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-07T01:22:55,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3848f991c723c9b72b68e9365062aeee, ASSIGN in 352 msec 2024-12-07T01:22:55,895 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server ec1863dc21e5,38575,1733534558286 after splitting done 2024-12-07T01:22:55,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure ec1863dc21e5,38575,1733534558286, splitWal=true, meta=false in 601 msec 2024-12-07T01:22:56,132 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee., hostname=ec1863dc21e5,35853,1733534558237, seqNum=18] 2024-12-07T01:22:56,151 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411 (was 414), OpenFileDescriptor=1051 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 309), ProcessCount=11 (was 11), AvailableMemoryMB=4890 (was 4933) 2024-12-07T01:22:56,153 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-07T01:22:56,166 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=411, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=4890 2024-12-07T01:22:56,166 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-07T01:22:56,182 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:56,184 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:56,185 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:56,187 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-68314103, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/hregion-68314103, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:56,198 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-68314103/hregion-68314103.1733534576188, exclude list is [], retry=0 2024-12-07T01:22:56,201 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:56,201 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:56,201 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:56,205 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-68314103/hregion-68314103.1733534576188 2024-12-07T01:22:56,205 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991)] 2024-12-07T01:22:56,206 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 634877e49662262b0720bf1e07b8196c, NAME => 'testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/hbase 2024-12-07T01:22:56,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741893_1071 (size=67) 2024-12-07T01:22:56,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741893_1071 (size=67) 2024-12-07T01:22:56,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741893_1071 (size=67) 2024-12-07T01:22:56,216 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:56,217 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,218 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634877e49662262b0720bf1e07b8196c columnFamilyName a 2024-12-07T01:22:56,219 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:56,219 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(327): Store=634877e49662262b0720bf1e07b8196c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:56,219 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,220 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634877e49662262b0720bf1e07b8196c columnFamilyName b 2024-12-07T01:22:56,220 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:56,221 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(327): Store=634877e49662262b0720bf1e07b8196c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:56,221 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,222 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634877e49662262b0720bf1e07b8196c columnFamilyName c 2024-12-07T01:22:56,222 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:56,222 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(327): Store=634877e49662262b0720bf1e07b8196c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:56,222 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,223 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,225 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,225 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,225 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:56,226 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,228 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:56,229 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 634877e49662262b0720bf1e07b8196c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70514670, jitterRate=0.05075046420097351}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:56,230 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 634877e49662262b0720bf1e07b8196c: Writing region info on filesystem at 1733534576216Initializing all the Stores at 1733534576217 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534576217Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534576217Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534576217Cleaning up temporary data from old regions at 1733534576225 (+8 ms)Region opened successfully at 1733534576230 (+5 ms) 2024-12-07T01:22:56,230 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 634877e49662262b0720bf1e07b8196c, disabling compactions & flushes 2024-12-07T01:22:56,230 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. 2024-12-07T01:22:56,231 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. 2024-12-07T01:22:56,231 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. after waiting 0 ms 2024-12-07T01:22:56,231 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. 2024-12-07T01:22:56,231 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. 2024-12-07T01:22:56,231 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 634877e49662262b0720bf1e07b8196c: Waiting for close lock at 1733534576230Disabling compacts and flushes for region at 1733534576230Disabling writes for close at 1733534576231 (+1 ms)Writing region close event to WAL at 1733534576231Closed at 1733534576231 2024-12-07T01:22:56,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741892_1070 (size=93) 2024-12-07T01:22:56,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741892_1070 (size=93) 2024-12-07T01:22:56,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741892_1070 (size=93) 2024-12-07T01:22:56,237 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:22:56,237 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-68314103:(num 1733534576188) 2024-12-07T01:22:56,237 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:56,239 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:56,250 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239, exclude list is [], retry=0 2024-12-07T01:22:56,253 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:56,253 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:56,253 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:56,255 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239 2024-12-07T01:22:56,255 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627)] 2024-12-07T01:22:56,255 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 634877e49662262b0720bf1e07b8196c, NAME => 'testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:56,256 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:56,256 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,256 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,257 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,258 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634877e49662262b0720bf1e07b8196c columnFamilyName a 2024-12-07T01:22:56,258 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:56,259 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(327): Store=634877e49662262b0720bf1e07b8196c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:56,259 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,260 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634877e49662262b0720bf1e07b8196c columnFamilyName b 2024-12-07T01:22:56,260 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:56,261 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(327): Store=634877e49662262b0720bf1e07b8196c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:56,261 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,261 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634877e49662262b0720bf1e07b8196c columnFamilyName c 2024-12-07T01:22:56,262 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:56,262 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(327): Store=634877e49662262b0720bf1e07b8196c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:56,262 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,263 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,266 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,266 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,266 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:56,268 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,269 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 634877e49662262b0720bf1e07b8196c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62332064, jitterRate=-0.07117986679077148}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:56,270 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 634877e49662262b0720bf1e07b8196c: Writing region info on filesystem at 1733534576256Initializing all the Stores at 1733534576257 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534576257Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534576257Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534576257Cleaning up temporary data from old regions at 1733534576266 (+9 ms)Region opened successfully at 1733534576270 (+4 ms) 2024-12-07T01:22:56,298 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 634877e49662262b0720bf1e07b8196c 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-07T01:22:56,314 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/a/836a47da2f484e7bbde86766d504cbb1 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733534576270/Put/seqid=0 2024-12-07T01:22:56,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741895_1073 (size=5958) 2024-12-07T01:22:56,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741895_1073 (size=5958) 2024-12-07T01:22:56,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741895_1073 (size=5958) 2024-12-07T01:22:56,322 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/a/836a47da2f484e7bbde86766d504cbb1 2024-12-07T01:22:56,341 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/b/fca466562ad4438f9f1ae15fa0eaae17 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733534576279/Put/seqid=0 2024-12-07T01:22:56,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741896_1074 (size=5958) 2024-12-07T01:22:56,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741896_1074 (size=5958) 2024-12-07T01:22:56,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741896_1074 (size=5958) 2024-12-07T01:22:56,752 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/b/fca466562ad4438f9f1ae15fa0eaae17 2024-12-07T01:22:56,862 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/c/77d779ef43f0442f8ea5dd796d251d6e is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733534576286/Put/seqid=0 2024-12-07T01:22:56,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741897_1075 (size=5958) 2024-12-07T01:22:56,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741897_1075 (size=5958) 2024-12-07T01:22:56,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741897_1075 (size=5958) 2024-12-07T01:22:56,869 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/c/77d779ef43f0442f8ea5dd796d251d6e 2024-12-07T01:22:56,875 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/a/836a47da2f484e7bbde86766d504cbb1 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/a/836a47da2f484e7bbde86766d504cbb1 2024-12-07T01:22:56,881 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/a/836a47da2f484e7bbde86766d504cbb1, entries=10, sequenceid=33, filesize=5.8 K 2024-12-07T01:22:56,882 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/b/fca466562ad4438f9f1ae15fa0eaae17 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/b/fca466562ad4438f9f1ae15fa0eaae17 2024-12-07T01:22:56,887 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/b/fca466562ad4438f9f1ae15fa0eaae17, entries=10, sequenceid=33, filesize=5.8 K 2024-12-07T01:22:56,888 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/c/77d779ef43f0442f8ea5dd796d251d6e as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/c/77d779ef43f0442f8ea5dd796d251d6e 2024-12-07T01:22:56,894 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/c/77d779ef43f0442f8ea5dd796d251d6e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-07T01:22:56,895 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 634877e49662262b0720bf1e07b8196c in 598ms, sequenceid=33, compaction requested=false 2024-12-07T01:22:56,895 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 634877e49662262b0720bf1e07b8196c: 2024-12-07T01:22:56,895 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 634877e49662262b0720bf1e07b8196c, disabling compactions & flushes 2024-12-07T01:22:56,895 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. 2024-12-07T01:22:56,895 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. 2024-12-07T01:22:56,896 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. after waiting 0 ms 2024-12-07T01:22:56,896 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. 2024-12-07T01:22:56,897 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. 2024-12-07T01:22:56,897 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 634877e49662262b0720bf1e07b8196c: Waiting for close lock at 1733534576895Disabling compacts and flushes for region at 1733534576895Disabling writes for close at 1733534576896 (+1 ms)Writing region close event to WAL at 1733534576897 (+1 ms)Closed at 1733534576897 2024-12-07T01:22:56,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741894_1072 (size=5937) 2024-12-07T01:22:56,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741894_1072 (size=5937) 2024-12-07T01:22:56,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741894_1072 (size=5937) 2024-12-07T01:22:56,904 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/b/fca466562ad4438f9f1ae15fa0eaae17 to hdfs://localhost:36673/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/b/fca466562ad4438f9f1ae15fa0eaae17 2024-12-07T01:22:56,919 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239, size=5.8 K (5937bytes) 2024-12-07T01:22:56,919 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239 2024-12-07T01:22:56,919 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239 after 0ms 2024-12-07T01:22:56,922 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239 took 3ms 2024-12-07T01:22:56,924 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239 so closing down 2024-12-07T01:22:56,924 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:22:56,925 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733534576239.temp 2024-12-07T01:22:56,926 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000003-wal.1733534576239.temp 2024-12-07T01:22:56,926 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:22:56,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741898_1076 (size=5223) 2024-12-07T01:22:56,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741898_1076 (size=5223) 2024-12-07T01:22:56,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741898_1076 (size=5223) 2024-12-07T01:22:56,933 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000003-wal.1733534576239.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-07T01:22:56,934 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000003-wal.1733534576239.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000032 2024-12-07T01:22:56,935 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-12-07T01:22:56,935 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239, journal: Splitting hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239, size=5.8 K (5937bytes) at 1733534576919Finishing writing output for hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239 so closing down at 1733534576924 (+5 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000003-wal.1733534576239.temp at 1733534576926 (+2 ms)3 split writer threads finished at 1733534576926Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000003-wal.1733534576239.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733534576933 (+7 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000003-wal.1733534576239.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000032 at 1733534576934 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1733534576935 (+1 ms) 2024-12-07T01:22:56,936 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576239 to hdfs://localhost:36673/hbase/oldWALs/wal.1733534576239 2024-12-07T01:22:56,937 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000032 2024-12-07T01:22:56,937 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:56,938 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:56,949 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576939, exclude list is [], retry=0 2024-12-07T01:22:56,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:56,953 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:56,953 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:56,956 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576939 2024-12-07T01:22:56,957 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627)] 2024-12-07T01:22:56,957 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 634877e49662262b0720bf1e07b8196c, NAME => 'testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:56,957 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:56,957 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,957 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,958 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,959 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634877e49662262b0720bf1e07b8196c columnFamilyName a 2024-12-07T01:22:56,959 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:56,965 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/a/836a47da2f484e7bbde86766d504cbb1 2024-12-07T01:22:56,965 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(327): Store=634877e49662262b0720bf1e07b8196c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:56,966 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,967 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634877e49662262b0720bf1e07b8196c columnFamilyName b 2024-12-07T01:22:56,967 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:56,968 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(327): Store=634877e49662262b0720bf1e07b8196c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:56,968 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,969 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 634877e49662262b0720bf1e07b8196c columnFamilyName c 2024-12-07T01:22:56,969 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:56,975 DEBUG [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/c/77d779ef43f0442f8ea5dd796d251d6e 2024-12-07T01:22:56,975 INFO [StoreOpener-634877e49662262b0720bf1e07b8196c-1 {}] regionserver.HStore(327): Store=634877e49662262b0720bf1e07b8196c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:56,975 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,976 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:56,978 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000032 2024-12-07T01:22:56,982 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000032 2024-12-07T01:22:56,982 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 634877e49662262b0720bf1e07b8196c 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-07T01:22:56,997 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/b/e4b98a98d9594694ac64bee71e427790 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733534576279/Put/seqid=0 2024-12-07T01:22:57,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741900_1078 (size=5958) 2024-12-07T01:22:57,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741900_1078 (size=5958) 2024-12-07T01:22:57,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741900_1078 (size=5958) 2024-12-07T01:22:57,003 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/b/e4b98a98d9594694ac64bee71e427790 2024-12-07T01:22:57,009 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/.tmp/b/e4b98a98d9594694ac64bee71e427790 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/b/e4b98a98d9594694ac64bee71e427790 2024-12-07T01:22:57,015 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/b/e4b98a98d9594694ac64bee71e427790, entries=10, sequenceid=32, filesize=5.8 K 2024-12-07T01:22:57,015 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 634877e49662262b0720bf1e07b8196c in 33ms, sequenceid=32, compaction requested=false; wal=null 2024-12-07T01:22:57,016 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/0000000000000000032 2024-12-07T01:22:57,017 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:57,017 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:57,018 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:57,020 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 634877e49662262b0720bf1e07b8196c 2024-12-07T01:22:57,022 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/634877e49662262b0720bf1e07b8196c/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-07T01:22:57,023 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 634877e49662262b0720bf1e07b8196c; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72478304, jitterRate=0.08001089096069336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:57,023 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 634877e49662262b0720bf1e07b8196c: Writing region info on filesystem at 1733534576957Initializing all the Stores at 1733534576958 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534576958Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534576958Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534576958Obtaining lock to block concurrent updates at 1733534576982 (+24 ms)Preparing flush snapshotting stores in 634877e49662262b0720bf1e07b8196c at 1733534576982Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733534576982Flushing stores of testReplayEditsWrittenViaHRegion,,1733534576183.634877e49662262b0720bf1e07b8196c. at 1733534576982Flushing 634877e49662262b0720bf1e07b8196c/b: creating writer at 1733534576982Flushing 634877e49662262b0720bf1e07b8196c/b: appending metadata at 1733534576996 (+14 ms)Flushing 634877e49662262b0720bf1e07b8196c/b: closing flushed file at 1733534576996Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b5db4bd: reopening flushed file at 1733534577008 (+12 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 634877e49662262b0720bf1e07b8196c in 33ms, sequenceid=32, compaction requested=false; wal=null at 1733534577015 (+7 ms)Cleaning up temporary data from old regions at 1733534577017 (+2 ms)Region opened successfully at 1733534577023 (+6 ms) 2024-12-07T01:22:57,042 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=419 (was 411) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:60424 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:58588 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:58414 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:42354 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:60256 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:42496 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1125 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 308), ProcessCount=11 (was 11), AvailableMemoryMB=4879 (was 4890) 2024-12-07T01:22:57,042 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-12-07T01:22:57,054 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419, OpenFileDescriptor=1125, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=4878 2024-12-07T01:22:57,054 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-12-07T01:22:57,069 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:57,071 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:22:57,072 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:22:57,074 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-65774850, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/hregion-65774850, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:57,085 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-65774850/hregion-65774850.1733534577075, exclude list is [], retry=0 2024-12-07T01:22:57,088 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:57,088 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:57,088 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:57,090 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-65774850/hregion-65774850.1733534577075 2024-12-07T01:22:57,090 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:57,090 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 35734e89439cd53acb40cd17d0913f96, NAME => 'testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/hbase 2024-12-07T01:22:57,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741902_1080 (size=68) 2024-12-07T01:22:57,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741902_1080 (size=68) 2024-12-07T01:22:57,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741902_1080 (size=68) 2024-12-07T01:22:57,098 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:57,099 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,101 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35734e89439cd53acb40cd17d0913f96 columnFamilyName a 2024-12-07T01:22:57,101 DEBUG [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:57,101 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(327): Store=35734e89439cd53acb40cd17d0913f96/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:57,101 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,102 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35734e89439cd53acb40cd17d0913f96 columnFamilyName b 2024-12-07T01:22:57,103 DEBUG [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:57,103 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(327): Store=35734e89439cd53acb40cd17d0913f96/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:57,103 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,104 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35734e89439cd53acb40cd17d0913f96 columnFamilyName c 2024-12-07T01:22:57,104 DEBUG [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:57,105 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(327): Store=35734e89439cd53acb40cd17d0913f96/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:57,105 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,106 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,106 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,107 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,107 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,108 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:57,109 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,111 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:22:57,112 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 35734e89439cd53acb40cd17d0913f96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58771409, jitterRate=-0.12423776090145111}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:57,112 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 35734e89439cd53acb40cd17d0913f96: Writing region info on filesystem at 1733534577098Initializing all the Stores at 1733534577099 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534577099Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534577099Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534577099Cleaning up temporary data from old regions at 1733534577107 (+8 ms)Region opened successfully at 1733534577112 (+5 ms) 2024-12-07T01:22:57,112 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 35734e89439cd53acb40cd17d0913f96, disabling compactions & flushes 2024-12-07T01:22:57,113 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. 2024-12-07T01:22:57,113 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. 2024-12-07T01:22:57,113 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. after waiting 0 ms 2024-12-07T01:22:57,113 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. 2024-12-07T01:22:57,113 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. 2024-12-07T01:22:57,113 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 35734e89439cd53acb40cd17d0913f96: Waiting for close lock at 1733534577112Disabling compacts and flushes for region at 1733534577112Disabling writes for close at 1733534577113 (+1 ms)Writing region close event to WAL at 1733534577113Closed at 1733534577113 2024-12-07T01:22:57,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741901_1079 (size=93) 2024-12-07T01:22:57,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741901_1079 (size=93) 2024-12-07T01:22:57,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741901_1079 (size=93) 2024-12-07T01:22:57,118 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:22:57,118 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-65774850:(num 1733534577075) 2024-12-07T01:22:57,118 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:22:57,120 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:22:57,130 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120, exclude list is [], retry=0 2024-12-07T01:22:57,132 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:22:57,133 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:22:57,133 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:22:57,134 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120 2024-12-07T01:22:57,134 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:22:57,181 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 35734e89439cd53acb40cd17d0913f96, NAME => 'testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:22:57,183 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,183 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:22:57,183 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,183 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,185 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,186 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35734e89439cd53acb40cd17d0913f96 columnFamilyName a 2024-12-07T01:22:57,186 DEBUG [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:57,187 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(327): Store=35734e89439cd53acb40cd17d0913f96/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:57,187 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,188 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35734e89439cd53acb40cd17d0913f96 columnFamilyName b 2024-12-07T01:22:57,188 DEBUG [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:57,188 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(327): Store=35734e89439cd53acb40cd17d0913f96/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:57,188 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,189 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35734e89439cd53acb40cd17d0913f96 columnFamilyName c 2024-12-07T01:22:57,189 DEBUG [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:22:57,190 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(327): Store=35734e89439cd53acb40cd17d0913f96/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:22:57,190 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,190 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,191 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,192 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,192 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,193 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:22:57,194 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,195 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 35734e89439cd53acb40cd17d0913f96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61328581, jitterRate=-0.08613292872905731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:22:57,195 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:22:57,195 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 35734e89439cd53acb40cd17d0913f96: Running coprocessor pre-open hook at 1733534577183Writing region info on filesystem at 1733534577183Initializing all the Stores at 1733534577184 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534577184Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534577185 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534577185Cleaning up temporary data from old regions at 1733534577192 (+7 ms)Running coprocessor post-open hooks at 1733534577195 (+3 ms)Region opened successfully at 1733534577195 2024-12-07T01:22:57,207 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 35734e89439cd53acb40cd17d0913f96 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-07T01:22:57,208 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 35734e89439cd53acb40cd17d0913f96/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:57,730 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-07T01:22:57,730 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-07T01:22:57,732 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-07T01:22:57,732 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-07T01:22:57,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-07T01:22:57,734 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-07T01:22:57,736 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-07T01:22:57,736 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-07T01:22:57,738 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-07T01:22:57,738 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-07T01:22:58,209 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 35734e89439cd53acb40cd17d0913f96/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:22:58,682 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T01:22:59,210 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 35734e89439cd53acb40cd17d0913f96/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:00,211 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 35734e89439cd53acb40cd17d0913f96/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:01,213 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 35734e89439cd53acb40cd17d0913f96/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:02,213 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 35734e89439cd53acb40cd17d0913f96/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:03,215 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 35734e89439cd53acb40cd17d0913f96/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:04,217 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 35734e89439cd53acb40cd17d0913f96/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:05,218 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 35734e89439cd53acb40cd17d0913f96/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:06,219 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 35734e89439cd53acb40cd17d0913f96/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:06,220 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 35734e89439cd53acb40cd17d0913f96: 2024-12-07T01:23:06,220 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. 2024-12-07T01:23:06,234 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 35734e89439cd53acb40cd17d0913f96: 2024-12-07T01:23:06,234 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-07T01:23:06,234 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 35734e89439cd53acb40cd17d0913f96, disabling compactions & flushes 2024-12-07T01:23:06,234 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. 2024-12-07T01:23:06,234 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. 2024-12-07T01:23:06,234 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. after waiting 0 ms 2024-12-07T01:23:06,234 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. 2024-12-07T01:23:06,235 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. 2024-12-07T01:23:06,235 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. 2024-12-07T01:23:06,235 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 35734e89439cd53acb40cd17d0913f96: Waiting for close lock at 1733534586234Running coprocessor pre-close hooks at 1733534586234Disabling compacts and flushes for region at 1733534586234Disabling writes for close at 1733534586234Writing region close event to WAL at 1733534586235 (+1 ms)Running coprocessor post-close hooks at 1733534586235Closed at 1733534586235 2024-12-07T01:23:06,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741903_1081 (size=3941) 2024-12-07T01:23:06,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741903_1081 (size=3941) 2024-12-07T01:23:06,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741903_1081 (size=3941) 2024-12-07T01:23:06,255 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120, size=3.8 K (3941bytes) 2024-12-07T01:23:06,255 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120 2024-12-07T01:23:06,255 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120 after 0ms 2024-12-07T01:23:06,259 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120 took 5ms 2024-12-07T01:23:06,261 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120 so closing down 2024-12-07T01:23:06,261 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:23:06,262 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733534577120.temp 2024-12-07T01:23:06,263 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000004-wal.1733534577120.temp 2024-12-07T01:23:06,264 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:23:06,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741904_1082 (size=2983) 2024-12-07T01:23:06,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741904_1082 (size=2983) 2024-12-07T01:23:06,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741904_1082 (size=2983) 2024-12-07T01:23:06,274 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000004-wal.1733534577120.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-07T01:23:06,276 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000004-wal.1733534577120.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000026 2024-12-07T01:23:06,276 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-12-07T01:23:06,276 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120, journal: Splitting hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120, size=3.8 K (3941bytes) at 1733534586255Finishing writing output for hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120 so closing down at 1733534586261 (+6 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000004-wal.1733534577120.temp at 1733534586263 (+2 ms)3 split writer threads finished at 1733534586264 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000004-wal.1733534577120.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733534586274 (+10 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000004-wal.1733534577120.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000026 at 1733534586276 (+2 ms)Processed 23 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1733534586276 2024-12-07T01:23:06,278 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534577120 to hdfs://localhost:36673/hbase/oldWALs/wal.1733534577120 2024-12-07T01:23:06,279 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000026 2024-12-07T01:23:06,280 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:23:06,282 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:06,299 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534586282, exclude list is [], retry=0 2024-12-07T01:23:06,302 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:06,303 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:06,303 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:06,304 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534586282 2024-12-07T01:23:06,305 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:23:06,306 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 35734e89439cd53acb40cd17d0913f96, NAME => 'testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:23:06,306 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,306 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:06,307 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,307 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,309 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,310 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35734e89439cd53acb40cd17d0913f96 columnFamilyName a 2024-12-07T01:23:06,310 DEBUG [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:06,311 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(327): Store=35734e89439cd53acb40cd17d0913f96/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:06,311 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,312 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35734e89439cd53acb40cd17d0913f96 columnFamilyName b 2024-12-07T01:23:06,312 DEBUG [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:06,312 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(327): Store=35734e89439cd53acb40cd17d0913f96/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:06,313 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,313 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35734e89439cd53acb40cd17d0913f96 columnFamilyName c 2024-12-07T01:23:06,313 DEBUG [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:06,314 INFO [StoreOpener-35734e89439cd53acb40cd17d0913f96-1 {}] regionserver.HStore(327): Store=35734e89439cd53acb40cd17d0913f96/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:06,314 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,315 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,316 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,316 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000026 2024-12-07T01:23:06,320 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000026 2024-12-07T01:23:06,320 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 35734e89439cd53acb40cd17d0913f96 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-07T01:23:06,333 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/.tmp/a/07da0da378dd40fd974d98c21631ba6d is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733534586225/Put/seqid=0 2024-12-07T01:23:06,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741906_1084 (size=5523) 2024-12-07T01:23:06,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741906_1084 (size=5523) 2024-12-07T01:23:06,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741906_1084 (size=5523) 2024-12-07T01:23:06,341 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/.tmp/a/07da0da378dd40fd974d98c21631ba6d 2024-12-07T01:23:06,358 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/.tmp/b/c81298272c524622a1f8b2879d37c764 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733534586221/Put/seqid=0 2024-12-07T01:23:06,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741907_1085 (size=5524) 2024-12-07T01:23:06,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741907_1085 (size=5524) 2024-12-07T01:23:06,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741907_1085 (size=5524) 2024-12-07T01:23:06,366 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/.tmp/b/c81298272c524622a1f8b2879d37c764 2024-12-07T01:23:06,387 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/.tmp/c/901c25906008496bb9a919ad4207e589 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733534586223/Put/seqid=0 2024-12-07T01:23:06,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741908_1086 (size=5457) 2024-12-07T01:23:06,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741908_1086 (size=5457) 2024-12-07T01:23:06,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741908_1086 (size=5457) 2024-12-07T01:23:06,395 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/.tmp/c/901c25906008496bb9a919ad4207e589 2024-12-07T01:23:06,403 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/.tmp/a/07da0da378dd40fd974d98c21631ba6d as hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/a/07da0da378dd40fd974d98c21631ba6d 2024-12-07T01:23:06,408 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/a/07da0da378dd40fd974d98c21631ba6d, entries=7, sequenceid=26, filesize=5.4 K 2024-12-07T01:23:06,409 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/.tmp/b/c81298272c524622a1f8b2879d37c764 as hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/b/c81298272c524622a1f8b2879d37c764 2024-12-07T01:23:06,414 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/b/c81298272c524622a1f8b2879d37c764, entries=7, sequenceid=26, filesize=5.4 K 2024-12-07T01:23:06,415 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/.tmp/c/901c25906008496bb9a919ad4207e589 as hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/c/901c25906008496bb9a919ad4207e589 2024-12-07T01:23:06,419 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/c/901c25906008496bb9a919ad4207e589, entries=6, sequenceid=26, filesize=5.3 K 2024-12-07T01:23:06,420 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 35734e89439cd53acb40cd17d0913f96 in 100ms, sequenceid=26, compaction requested=false; wal=null 2024-12-07T01:23:06,421 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/0000000000000000026 2024-12-07T01:23:06,422 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,422 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,422 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:23:06,424 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,426 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsAfterAbortingFlush/35734e89439cd53acb40cd17d0913f96/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-07T01:23:06,427 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 35734e89439cd53acb40cd17d0913f96; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71599681, jitterRate=0.06691838800907135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:23:06,427 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 35734e89439cd53acb40cd17d0913f96 2024-12-07T01:23:06,428 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 35734e89439cd53acb40cd17d0913f96: Running coprocessor pre-open hook at 1733534586307Writing region info on filesystem at 1733534586307Initializing all the Stores at 1733534586308 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534586308Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534586308Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534586308Obtaining lock to block concurrent updates at 1733534586320 (+12 ms)Preparing flush snapshotting stores in 35734e89439cd53acb40cd17d0913f96 at 1733534586320Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733534586321 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1733534577070.35734e89439cd53acb40cd17d0913f96. at 1733534586321Flushing 35734e89439cd53acb40cd17d0913f96/a: creating writer at 1733534586321Flushing 35734e89439cd53acb40cd17d0913f96/a: appending metadata at 1733534586333 (+12 ms)Flushing 35734e89439cd53acb40cd17d0913f96/a: closing flushed file at 1733534586333Flushing 35734e89439cd53acb40cd17d0913f96/b: creating writer at 1733534586345 (+12 ms)Flushing 35734e89439cd53acb40cd17d0913f96/b: appending metadata at 1733534586358 (+13 ms)Flushing 35734e89439cd53acb40cd17d0913f96/b: closing flushed file at 1733534586358Flushing 35734e89439cd53acb40cd17d0913f96/c: creating writer at 1733534586371 (+13 ms)Flushing 35734e89439cd53acb40cd17d0913f96/c: appending metadata at 1733534586387 (+16 ms)Flushing 35734e89439cd53acb40cd17d0913f96/c: closing flushed file at 1733534586387Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bdbe2c9: reopening flushed file at 1733534586402 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bf20593: reopening flushed file at 1733534586408 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e6d516b: reopening flushed file at 1733534586414 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 35734e89439cd53acb40cd17d0913f96 in 100ms, sequenceid=26, compaction requested=false; wal=null at 1733534586420 (+6 ms)Cleaning up temporary data from old regions at 1733534586422 (+2 ms)Running coprocessor post-open hooks at 1733534586427 (+5 ms)Region opened successfully at 1733534586428 (+1 ms) 2024-12-07T01:23:06,448 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=421 (was 419) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:42828 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:32944 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:47068 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:47086 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:42856 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:32938 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1183 (was 1125) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 308), ProcessCount=11 (was 11), AvailableMemoryMB=4864 (was 4878) 2024-12-07T01:23:06,448 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-07T01:23:06,462 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=421, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=4863 2024-12-07T01:23:06,462 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-07T01:23:06,481 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:23:06,483 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:23:06,484 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:23:06,487 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-29279544, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/hregion-29279544, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:06,503 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-29279544/hregion-29279544.1733534586487, exclude list is [], retry=0 2024-12-07T01:23:06,506 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:06,506 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:06,506 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:06,508 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-29279544/hregion-29279544.1733534586487 2024-12-07T01:23:06,508 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991)] 2024-12-07T01:23:06,508 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 86500fa4e547eb03968d6c2374ee94e2, NAME => 'testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/hbase 2024-12-07T01:23:06,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741910_1088 (size=61) 2024-12-07T01:23:06,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741910_1088 (size=61) 2024-12-07T01:23:06,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741910_1088 (size=61) 2024-12-07T01:23:06,517 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:06,518 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,519 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86500fa4e547eb03968d6c2374ee94e2 columnFamilyName a 2024-12-07T01:23:06,519 DEBUG [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:06,520 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] regionserver.HStore(327): Store=86500fa4e547eb03968d6c2374ee94e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:06,520 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,521 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,521 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,521 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,521 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,523 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,525 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:23:06,525 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 86500fa4e547eb03968d6c2374ee94e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71354386, jitterRate=0.06326320767402649}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:23:06,526 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 86500fa4e547eb03968d6c2374ee94e2: Writing region info on filesystem at 1733534586517Initializing all the Stores at 1733534586518 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534586518Cleaning up temporary data from old regions at 1733534586521 (+3 ms)Region opened successfully at 1733534586526 (+5 ms) 2024-12-07T01:23:06,526 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 86500fa4e547eb03968d6c2374ee94e2, disabling compactions & flushes 2024-12-07T01:23:06,526 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. 2024-12-07T01:23:06,526 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. 2024-12-07T01:23:06,526 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. after waiting 0 ms 2024-12-07T01:23:06,526 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. 2024-12-07T01:23:06,527 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. 2024-12-07T01:23:06,527 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 86500fa4e547eb03968d6c2374ee94e2: Waiting for close lock at 1733534586526Disabling compacts and flushes for region at 1733534586526Disabling writes for close at 1733534586526Writing region close event to WAL at 1733534586527 (+1 ms)Closed at 1733534586527 2024-12-07T01:23:06,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741909_1087 (size=93) 2024-12-07T01:23:06,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741909_1087 (size=93) 2024-12-07T01:23:06,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741909_1087 (size=93) 2024-12-07T01:23:06,533 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:23:06,533 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-29279544:(num 1733534586487) 2024-12-07T01:23:06,533 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:23:06,535 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:06,539 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T01:23:06,546 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535, exclude list is [], retry=0 2024-12-07T01:23:06,548 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:06,548 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:06,549 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:06,550 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535 2024-12-07T01:23:06,550 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627)] 2024-12-07T01:23:06,550 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 86500fa4e547eb03968d6c2374ee94e2, NAME => 'testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:23:06,551 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:06,551 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,551 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,552 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,553 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86500fa4e547eb03968d6c2374ee94e2 columnFamilyName a 2024-12-07T01:23:06,553 DEBUG [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:06,553 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] regionserver.HStore(327): Store=86500fa4e547eb03968d6c2374ee94e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:06,553 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,554 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,555 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,556 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,556 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,557 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,558 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 86500fa4e547eb03968d6c2374ee94e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66961989, jitterRate=-0.0021886080503463745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:23:06,559 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 86500fa4e547eb03968d6c2374ee94e2: Writing region info on filesystem at 1733534586551Initializing all the Stores at 1733534586552 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534586552Cleaning up temporary data from old regions at 1733534586556 (+4 ms)Region opened successfully at 1733534586559 (+3 ms) 2024-12-07T01:23:06,567 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 86500fa4e547eb03968d6c2374ee94e2, disabling compactions & flushes 2024-12-07T01:23:06,567 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. 2024-12-07T01:23:06,567 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. 2024-12-07T01:23:06,567 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. after waiting 0 ms 2024-12-07T01:23:06,567 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. 2024-12-07T01:23:06,567 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. 2024-12-07T01:23:06,567 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. 2024-12-07T01:23:06,567 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 86500fa4e547eb03968d6c2374ee94e2: Waiting for close lock at 1733534586567Disabling compacts and flushes for region at 1733534586567Disabling writes for close at 1733534586567Writing region close event to WAL at 1733534586567Closed at 1733534586567 2024-12-07T01:23:06,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741911_1089 (size=1623) 2024-12-07T01:23:06,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741911_1089 (size=1623) 2024-12-07T01:23:06,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741911_1089 (size=1623) 2024-12-07T01:23:06,584 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535, size=1.6 K (1623bytes) 2024-12-07T01:23:06,584 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535 2024-12-07T01:23:06,585 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535 after 1ms 2024-12-07T01:23:06,587 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535 took 3ms 2024-12-07T01:23:06,588 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535 so closing down 2024-12-07T01:23:06,588 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:23:06,589 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733534586535.temp 2024-12-07T01:23:06,590 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000003-wal.1733534586535.temp 2024-12-07T01:23:06,591 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:23:06,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741912_1090 (size=1623) 2024-12-07T01:23:06,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741912_1090 (size=1623) 2024-12-07T01:23:06,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741912_1090 (size=1623) 2024-12-07T01:23:06,597 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000003-wal.1733534586535.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-07T01:23:06,599 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000003-wal.1733534586535.temp to hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000012 2024-12-07T01:23:06,599 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-12-07T01:23:06,599 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535, journal: Splitting hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535, size=1.6 K (1623bytes) at 1733534586584Finishing writing output for hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535 so closing down at 1733534586588 (+4 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000003-wal.1733534586535.temp at 1733534586590 (+2 ms)3 split writer threads finished at 1733534586591 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000003-wal.1733534586535.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733534586597 (+6 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000003-wal.1733534586535.temp to hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000012 at 1733534586599 (+2 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1733534586599 2024-12-07T01:23:06,601 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586535 to hdfs://localhost:36673/hbase/oldWALs/wal.1733534586535 2024-12-07T01:23:06,602 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000012 2024-12-07T01:23:06,897 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:23:06,900 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:06,910 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586900, exclude list is [], retry=0 2024-12-07T01:23:06,913 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:06,913 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:06,914 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:06,915 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586900 2024-12-07T01:23:06,915 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:23:06,915 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 86500fa4e547eb03968d6c2374ee94e2, NAME => 'testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:23:06,916 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:06,916 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,916 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,918 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,920 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86500fa4e547eb03968d6c2374ee94e2 columnFamilyName a 2024-12-07T01:23:06,920 DEBUG [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:06,921 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] regionserver.HStore(327): Store=86500fa4e547eb03968d6c2374ee94e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:06,921 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,922 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,923 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,924 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000012 2024-12-07T01:23:06,927 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000012 2024-12-07T01:23:06,927 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 86500fa4e547eb03968d6c2374ee94e2 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-07T01:23:06,944 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/.tmp/a/83accaf0124a439bbd33ad8dec91b449 is 79, key is testDatalossWhenInputError/a:x0/1733534586559/Put/seqid=0 2024-12-07T01:23:06,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741914_1092 (size=5808) 2024-12-07T01:23:06,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741914_1092 (size=5808) 2024-12-07T01:23:06,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741914_1092 (size=5808) 2024-12-07T01:23:06,954 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/.tmp/a/83accaf0124a439bbd33ad8dec91b449 2024-12-07T01:23:06,966 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/.tmp/a/83accaf0124a439bbd33ad8dec91b449 as hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/a/83accaf0124a439bbd33ad8dec91b449 2024-12-07T01:23:06,975 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/a/83accaf0124a439bbd33ad8dec91b449, entries=10, sequenceid=12, filesize=5.7 K 2024-12-07T01:23:06,975 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 86500fa4e547eb03968d6c2374ee94e2 in 48ms, sequenceid=12, compaction requested=false; wal=null 2024-12-07T01:23:06,976 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/0000000000000000012 2024-12-07T01:23:06,977 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,977 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,980 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,982 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-07T01:23:06,983 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 86500fa4e547eb03968d6c2374ee94e2; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71335544, jitterRate=0.06298243999481201}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:23:06,984 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 86500fa4e547eb03968d6c2374ee94e2: Writing region info on filesystem at 1733534586916Initializing all the Stores at 1733534586918 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534586918Obtaining lock to block concurrent updates at 1733534586927 (+9 ms)Preparing flush snapshotting stores in 86500fa4e547eb03968d6c2374ee94e2 at 1733534586927Finished memstore snapshotting testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733534586928 (+1 ms)Flushing stores of testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2. at 1733534586928Flushing 86500fa4e547eb03968d6c2374ee94e2/a: creating writer at 1733534586928Flushing 86500fa4e547eb03968d6c2374ee94e2/a: appending metadata at 1733534586944 (+16 ms)Flushing 86500fa4e547eb03968d6c2374ee94e2/a: closing flushed file at 1733534586944Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b78d4d5: reopening flushed file at 1733534586964 (+20 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 86500fa4e547eb03968d6c2374ee94e2 in 48ms, sequenceid=12, compaction requested=false; wal=null at 1733534586975 (+11 ms)Cleaning up temporary data from old regions at 1733534586977 (+2 ms)Region opened successfully at 1733534586984 (+7 ms) 2024-12-07T01:23:06,987 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 86500fa4e547eb03968d6c2374ee94e2, NAME => 'testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:23:06,987 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733534586481.86500fa4e547eb03968d6c2374ee94e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:06,987 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,987 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,989 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,990 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86500fa4e547eb03968d6c2374ee94e2 columnFamilyName a 2024-12-07T01:23:06,990 DEBUG [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:06,996 DEBUG [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/a/83accaf0124a439bbd33ad8dec91b449 2024-12-07T01:23:06,996 INFO [StoreOpener-86500fa4e547eb03968d6c2374ee94e2-1 {}] regionserver.HStore(327): Store=86500fa4e547eb03968d6c2374ee94e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:06,996 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,997 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,999 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,999 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:06,999 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:07,001 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 86500fa4e547eb03968d6c2374ee94e2 2024-12-07T01:23:07,003 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testDatalossWhenInputError/86500fa4e547eb03968d6c2374ee94e2/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-07T01:23:07,004 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 86500fa4e547eb03968d6c2374ee94e2; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68013089, jitterRate=0.013474002480506897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T01:23:07,004 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 86500fa4e547eb03968d6c2374ee94e2: Writing region info on filesystem at 1733534586987Initializing all the Stores at 1733534586988 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534586988Cleaning up temporary data from old regions at 1733534586999 (+11 ms)Region opened successfully at 1733534587004 (+5 ms) 2024-12-07T01:23:07,020 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=430 (was 421) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:42828 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:33028 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:32944 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:47140 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:42920 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:47086 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-773003372-172.17.0.3-1733534552230:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1265 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=4858 (was 4863) 2024-12-07T01:23:07,020 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1265 is superior to 1024 2024-12-07T01:23:07,030 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=430, OpenFileDescriptor=1265, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=4857 2024-12-07T01:23:07,031 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1265 is superior to 1024 2024-12-07T01:23:07,044 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:23:07,045 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:23:07,046 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:23:07,049 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72950285, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/hregion-72950285, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:07,067 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72950285/hregion-72950285.1733534587049, exclude list is [], retry=0 2024-12-07T01:23:07,070 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:07,070 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:07,071 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:07,076 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72950285/hregion-72950285.1733534587049 2024-12-07T01:23:07,080 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:23:07,080 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6a2669c7abd67e34e9fcc0444798eaa5, NAME => 'testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/hbase 2024-12-07T01:23:07,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741916_1094 (size=63) 2024-12-07T01:23:07,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741916_1094 (size=63) 2024-12-07T01:23:07,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741916_1094 (size=63) 2024-12-07T01:23:07,094 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:07,096 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,098 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2669c7abd67e34e9fcc0444798eaa5 columnFamilyName a 2024-12-07T01:23:07,098 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:07,098 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(327): Store=6a2669c7abd67e34e9fcc0444798eaa5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:07,098 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,100 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2669c7abd67e34e9fcc0444798eaa5 columnFamilyName b 2024-12-07T01:23:07,100 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:07,100 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(327): Store=6a2669c7abd67e34e9fcc0444798eaa5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:07,100 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,102 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2669c7abd67e34e9fcc0444798eaa5 columnFamilyName c 2024-12-07T01:23:07,102 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:07,102 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(327): Store=6a2669c7abd67e34e9fcc0444798eaa5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:07,103 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,104 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,104 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,105 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,105 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,106 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:23:07,107 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,109 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:23:07,110 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a2669c7abd67e34e9fcc0444798eaa5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68667300, jitterRate=0.023222506046295166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:23:07,110 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a2669c7abd67e34e9fcc0444798eaa5: Writing region info on filesystem at 1733534587094Initializing all the Stores at 1733534587095 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534587095Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534587096 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534587096Cleaning up temporary data from old regions at 1733534587105 (+9 ms)Region opened successfully at 1733534587110 (+5 ms) 2024-12-07T01:23:07,111 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a2669c7abd67e34e9fcc0444798eaa5, disabling compactions & flushes 2024-12-07T01:23:07,111 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. 2024-12-07T01:23:07,111 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. 2024-12-07T01:23:07,111 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. after waiting 0 ms 2024-12-07T01:23:07,111 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. 2024-12-07T01:23:07,111 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. 2024-12-07T01:23:07,111 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a2669c7abd67e34e9fcc0444798eaa5: Waiting for close lock at 1733534587111Disabling compacts and flushes for region at 1733534587111Disabling writes for close at 1733534587111Writing region close event to WAL at 1733534587111Closed at 1733534587111 2024-12-07T01:23:07,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741915_1093 (size=93) 2024-12-07T01:23:07,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741915_1093 (size=93) 2024-12-07T01:23:07,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741915_1093 (size=93) 2024-12-07T01:23:07,116 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:23:07,116 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72950285:(num 1733534587049) 2024-12-07T01:23:07,117 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:23:07,118 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:07,131 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119, exclude list is [], retry=0 2024-12-07T01:23:07,134 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:07,135 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:07,135 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:07,136 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 2024-12-07T01:23:07,137 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:23:07,137 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a2669c7abd67e34e9fcc0444798eaa5, NAME => 'testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:23:07,137 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:07,137 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,137 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,138 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,139 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2669c7abd67e34e9fcc0444798eaa5 columnFamilyName a 2024-12-07T01:23:07,139 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:07,140 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(327): Store=6a2669c7abd67e34e9fcc0444798eaa5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:07,140 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,140 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2669c7abd67e34e9fcc0444798eaa5 columnFamilyName b 2024-12-07T01:23:07,140 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:07,141 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(327): Store=6a2669c7abd67e34e9fcc0444798eaa5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:07,141 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,142 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2669c7abd67e34e9fcc0444798eaa5 columnFamilyName c 2024-12-07T01:23:07,142 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:07,142 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(327): Store=6a2669c7abd67e34e9fcc0444798eaa5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:07,142 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,143 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,144 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,145 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,145 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,145 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:23:07,147 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:07,147 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a2669c7abd67e34e9fcc0444798eaa5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70990622, jitterRate=0.05784270167350769}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:23:07,148 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a2669c7abd67e34e9fcc0444798eaa5: Writing region info on filesystem at 1733534587137Initializing all the Stores at 1733534587138 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534587138Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534587138Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534587138Cleaning up temporary data from old regions at 1733534587145 (+7 ms)Region opened successfully at 1733534587148 (+3 ms) 2024-12-07T01:23:07,151 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733534587151/Put/seqid=0 2024-12-07T01:23:07,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741918_1096 (size=4875) 2024-12-07T01:23:07,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741918_1096 (size=4875) 2024-12-07T01:23:07,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741918_1096 (size=4875) 2024-12-07T01:23:07,161 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733534587161/Put/seqid=0 2024-12-07T01:23:07,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741919_1097 (size=4875) 2024-12-07T01:23:07,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741919_1097 (size=4875) 2024-12-07T01:23:07,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741919_1097 (size=4875) 2024-12-07T01:23:07,173 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733534587173/Put/seqid=0 2024-12-07T01:23:07,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741920_1098 (size=4875) 2024-12-07T01:23:07,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741920_1098 (size=4875) 2024-12-07T01:23:07,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741920_1098 (size=4875) 2024-12-07T01:23:07,181 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 6a2669c7abd67e34e9fcc0444798eaa5/a 2024-12-07T01:23:07,185 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-07T01:23:07,185 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-07T01:23:07,185 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 6a2669c7abd67e34e9fcc0444798eaa5/a 2024-12-07T01:23:07,189 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-07T01:23:07,189 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-07T01:23:07,189 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 6a2669c7abd67e34e9fcc0444798eaa5/a 2024-12-07T01:23:07,193 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-07T01:23:07,193 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-07T01:23:07,193 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a2669c7abd67e34e9fcc0444798eaa5 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-07T01:23:07,208 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/.tmp/a/cd73dfa9df8247bc97a27655170cda43 is 55, key is testCompactedBulkLoadedFiles/a:a/1733534587148/Put/seqid=0 2024-12-07T01:23:07,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741921_1099 (size=5107) 2024-12-07T01:23:07,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741921_1099 (size=5107) 2024-12-07T01:23:07,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741921_1099 (size=5107) 2024-12-07T01:23:07,215 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/.tmp/a/cd73dfa9df8247bc97a27655170cda43 2024-12-07T01:23:07,220 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/.tmp/a/cd73dfa9df8247bc97a27655170cda43 as hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/cd73dfa9df8247bc97a27655170cda43 2024-12-07T01:23:07,225 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/cd73dfa9df8247bc97a27655170cda43, entries=1, sequenceid=4, filesize=5.0 K 2024-12-07T01:23:07,226 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 6a2669c7abd67e34e9fcc0444798eaa5 in 33ms, sequenceid=4, compaction requested=false 2024-12-07T01:23:07,226 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6a2669c7abd67e34e9fcc0444798eaa5: 2024-12-07T01:23:07,228 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/a667540353f24437b4085d63cd57c416_SeqId_4_ 2024-12-07T01:23:07,229 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_ 2024-12-07T01:23:07,231 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_ 2024-12-07T01:23:07,231 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile0 into 6a2669c7abd67e34e9fcc0444798eaa5/a as hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/a667540353f24437b4085d63cd57c416_SeqId_4_ - updating store file list. 2024-12-07T01:23:07,236 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for a667540353f24437b4085d63cd57c416_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-07T01:23:07,236 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/a667540353f24437b4085d63cd57c416_SeqId_4_ into 6a2669c7abd67e34e9fcc0444798eaa5/a 2024-12-07T01:23:07,236 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile0 into 6a2669c7abd67e34e9fcc0444798eaa5/a (new location: hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/a667540353f24437b4085d63cd57c416_SeqId_4_) 2024-12-07T01:23:07,237 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile1 into 6a2669c7abd67e34e9fcc0444798eaa5/a as hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_ - updating store file list. 2024-12-07T01:23:07,242 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-07T01:23:07,242 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_ into 6a2669c7abd67e34e9fcc0444798eaa5/a 2024-12-07T01:23:07,242 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile1 into 6a2669c7abd67e34e9fcc0444798eaa5/a (new location: hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_) 2024-12-07T01:23:07,243 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile2 into 6a2669c7abd67e34e9fcc0444798eaa5/a as hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_ - updating store file list. 2024-12-07T01:23:07,247 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-07T01:23:07,247 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_ into 6a2669c7abd67e34e9fcc0444798eaa5/a 2024-12-07T01:23:07,247 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36673/hbase/testCompactedBulkLoadedFiles/hfile2 into 6a2669c7abd67e34e9fcc0444798eaa5/a (new location: hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_) 2024-12-07T01:23:07,253 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T01:23:07,254 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6a2669c7abd67e34e9fcc0444798eaa5/a is initiating major compaction (all files) 2024-12-07T01:23:07,254 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6a2669c7abd67e34e9fcc0444798eaa5/a in testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. 2024-12-07T01:23:07,254 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/cd73dfa9df8247bc97a27655170cda43, hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/a667540353f24437b4085d63cd57c416_SeqId_4_, hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_, hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_] into tmpdir=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/.tmp, totalSize=19.3 K 2024-12-07T01:23:07,254 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cd73dfa9df8247bc97a27655170cda43, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733534587148 2024-12-07T01:23:07,255 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a667540353f24437b4085d63cd57c416_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-07T01:23:07,255 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-07T01:23:07,255 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-07T01:23:07,267 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/.tmp/a/dbb22bd454574370985072943e658c16 is 55, key is testCompactedBulkLoadedFiles/a:a/1733534587148/Put/seqid=0 2024-12-07T01:23:07,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741922_1100 (size=6154) 2024-12-07T01:23:07,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741922_1100 (size=6154) 2024-12-07T01:23:07,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741922_1100 (size=6154) 2024-12-07T01:23:07,286 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/.tmp/a/dbb22bd454574370985072943e658c16 as hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/dbb22bd454574370985072943e658c16 2024-12-07T01:23:07,291 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 6a2669c7abd67e34e9fcc0444798eaa5/a of 6a2669c7abd67e34e9fcc0444798eaa5 into dbb22bd454574370985072943e658c16(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T01:23:07,291 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6a2669c7abd67e34e9fcc0444798eaa5: 2024-12-07T01:23:07,292 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-07T01:23:07,292 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-07T01:23:07,318 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119, size=0 (0bytes) 2024-12-07T01:23:07,319 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 might be still open, length is 0 2024-12-07T01:23:07,319 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 2024-12-07T01:23:07,319 WARN [IPC Server handler 2 on default port 36673 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-07T01:23:07,319 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 after 0ms 2024-12-07T01:23:07,730 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-07T01:23:07,730 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-07T01:23:07,732 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-07T01:23:07,732 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-07T01:23:09,500 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:42940 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45293:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42940 dst: /127.0.0.1:45293 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45293 remote=/127.0.0.1:42940]. Total timeout mills is 60000, 57791 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:09,501 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:33058 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:46343:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33058 dst: /127.0.0.1:46343 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:09,501 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:47168 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33463:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47168 dst: /127.0.0.1:33463 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:09,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741917_1101 (size=1659) 2024-12-07T01:23:09,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741917_1101 (size=1659) 2024-12-07T01:23:11,321 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 after 4001ms 2024-12-07T01:23:11,326 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 took 4008ms 2024-12-07T01:23:11,329 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119; continuing. 2024-12-07T01:23:11,329 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 so closing down 2024-12-07T01:23:11,329 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:23:11,331 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733534587119.temp 2024-12-07T01:23:11,333 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000003-wal.1733534587119.temp 2024-12-07T01:23:11,333 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:23:11,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741923_1102 (size=698) 2024-12-07T01:23:11,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741923_1102 (size=698) 2024-12-07T01:23:11,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741923_1102 (size=698) 2024-12-07T01:23:11,344 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000003-wal.1733534587119.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-07T01:23:11,346 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000003-wal.1733534587119.temp to hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000008 2024-12-07T01:23:11,346 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119, size=0, length=0, corrupted=false, cancelled=false 2024-12-07T01:23:11,346 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119, journal: Splitting hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119, size=0 (0bytes) at 1733534587318Finishing writing output for hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 so closing down at 1733534591329 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000003-wal.1733534587119.temp at 1733534591333 (+4 ms)3 split writer threads finished at 1733534591333Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000003-wal.1733534587119.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733534591344 (+11 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000003-wal.1733534587119.temp to hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000008 at 1733534591346 (+2 ms)Processed 5 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119, size=0, length=0, corrupted=false, cancelled=false at 1733534591346 2024-12-07T01:23:11,348 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 to hdfs://localhost:36673/hbase/oldWALs/wal.1733534587119 2024-12-07T01:23:11,350 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000008 2024-12-07T01:23:11,350 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:23:11,352 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:11,367 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534591352, exclude list is [], retry=0 2024-12-07T01:23:11,369 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:11,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:11,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:11,372 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534591352 2024-12-07T01:23:11,372 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627)] 2024-12-07T01:23:11,372 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a2669c7abd67e34e9fcc0444798eaa5, NAME => 'testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:23:11,372 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:11,372 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,372 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,374 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,374 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2669c7abd67e34e9fcc0444798eaa5 columnFamilyName a 2024-12-07T01:23:11,374 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,380 DEBUG [StoreFileOpener-6a2669c7abd67e34e9fcc0444798eaa5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-07T01:23:11,380 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_ 2024-12-07T01:23:11,384 DEBUG [StoreFileOpener-6a2669c7abd67e34e9fcc0444798eaa5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for a667540353f24437b4085d63cd57c416_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-07T01:23:11,384 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/a667540353f24437b4085d63cd57c416_SeqId_4_ 2024-12-07T01:23:11,387 DEBUG [StoreFileOpener-6a2669c7abd67e34e9fcc0444798eaa5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-07T01:23:11,388 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_ 2024-12-07T01:23:11,392 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/cd73dfa9df8247bc97a27655170cda43 2024-12-07T01:23:11,395 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/dbb22bd454574370985072943e658c16 2024-12-07T01:23:11,395 WARN [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@21f8b62b 2024-12-07T01:23:11,395 WARN [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/a667540353f24437b4085d63cd57c416_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@21f8b62b 2024-12-07T01:23:11,395 WARN [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@21f8b62b 2024-12-07T01:23:11,395 WARN [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/cd73dfa9df8247bc97a27655170cda43 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@21f8b62b 2024-12-07T01:23:11,395 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_, hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/a667540353f24437b4085d63cd57c416_SeqId_4_, hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_, hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/cd73dfa9df8247bc97a27655170cda43] to archive 2024-12-07T01:23:11,396 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T01:23:11,398 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_ to hdfs://localhost:36673/hbase/archive/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_ 2024-12-07T01:23:11,399 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/a667540353f24437b4085d63cd57c416_SeqId_4_ to hdfs://localhost:36673/hbase/archive/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/a667540353f24437b4085d63cd57c416_SeqId_4_ 2024-12-07T01:23:11,400 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_ to hdfs://localhost:36673/hbase/archive/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_ 2024-12-07T01:23:11,401 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/cd73dfa9df8247bc97a27655170cda43 to hdfs://localhost:36673/hbase/archive/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/a/cd73dfa9df8247bc97a27655170cda43 2024-12-07T01:23:11,401 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(327): Store=6a2669c7abd67e34e9fcc0444798eaa5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,401 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,402 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2669c7abd67e34e9fcc0444798eaa5 columnFamilyName b 2024-12-07T01:23:11,402 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,402 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(327): Store=6a2669c7abd67e34e9fcc0444798eaa5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,403 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,403 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2669c7abd67e34e9fcc0444798eaa5 columnFamilyName c 2024-12-07T01:23:11,403 DEBUG [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,404 INFO [StoreOpener-6a2669c7abd67e34e9fcc0444798eaa5-1 {}] regionserver.HStore(327): Store=6a2669c7abd67e34e9fcc0444798eaa5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,404 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,405 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,406 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,406 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000008 2024-12-07T01:23:11,409 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 6a2669c7abd67e34e9fcc0444798eaa5 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "6a2669c7abd67e34e9fcc0444798eaa5" family_name: "a" compaction_input: "cd73dfa9df8247bc97a27655170cda43" compaction_input: "a667540353f24437b4085d63cd57c416_SeqId_4_" compaction_input: "bf9a70dbb4a84ae4adca0ce79be7c860_SeqId_4_" compaction_input: "76109bf00b0e46ac88dc72c92a67a9a9_SeqId_4_" compaction_output: "dbb22bd454574370985072943e658c16" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-07T01:23:11,409 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-07T01:23:11,409 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000008 2024-12-07T01:23:11,410 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/0000000000000000008 2024-12-07T01:23:11,411 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,411 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,411 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:23:11,413 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a2669c7abd67e34e9fcc0444798eaa5 2024-12-07T01:23:11,415 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testCompactedBulkLoadedFiles/6a2669c7abd67e34e9fcc0444798eaa5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-07T01:23:11,416 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a2669c7abd67e34e9fcc0444798eaa5; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74897996, jitterRate=0.11606711149215698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:23:11,416 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a2669c7abd67e34e9fcc0444798eaa5: Writing region info on filesystem at 1733534591373Initializing all the Stores at 1733534591373Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591373Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591373Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591373Cleaning up temporary data from old regions at 1733534591411 (+38 ms)Region opened successfully at 1733534591416 (+5 ms) 2024-12-07T01:23:11,418 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a2669c7abd67e34e9fcc0444798eaa5, disabling compactions & flushes 2024-12-07T01:23:11,418 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. 2024-12-07T01:23:11,418 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. 2024-12-07T01:23:11,418 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. after waiting 0 ms 2024-12-07T01:23:11,418 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. 2024-12-07T01:23:11,419 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733534587044.6a2669c7abd67e34e9fcc0444798eaa5. 2024-12-07T01:23:11,419 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a2669c7abd67e34e9fcc0444798eaa5: Waiting for close lock at 1733534591418Disabling compacts and flushes for region at 1733534591418Disabling writes for close at 1733534591418Writing region close event to WAL at 1733534591419 (+1 ms)Closed at 1733534591419 2024-12-07T01:23:11,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741924_1103 (size=93) 2024-12-07T01:23:11,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741924_1103 (size=93) 2024-12-07T01:23:11,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741924_1103 (size=93) 2024-12-07T01:23:11,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:23:11,423 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733534591352) 2024-12-07T01:23:11,438 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=443 (was 430) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_424243126_22 at /127.0.0.1:42992 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:36673 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:36673 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_424243126_22 at /127.0.0.1:47284 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1347 (was 1265) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=240 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=4845 (was 4857) 2024-12-07T01:23:11,438 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1347 is superior to 1024 2024-12-07T01:23:11,449 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1347, MaxFileDescriptor=1048576, SystemLoadAverage=240, ProcessCount=11, AvailableMemoryMB=4843 2024-12-07T01:23:11,449 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1347 is superior to 1024 2024-12-07T01:23:11,462 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:23:11,463 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T01:23:11,464 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T01:23:11,466 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99618403, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/hregion-99618403, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:11,478 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99618403/hregion-99618403.1733534591466, exclude list is [], retry=0 2024-12-07T01:23:11,480 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:11,481 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:11,481 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:11,483 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99618403/hregion-99618403.1733534591466 2024-12-07T01:23:11,483 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34991:34991),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753)] 2024-12-07T01:23:11,483 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e9f140ba5e4bd358e794cc587c08a6f0, NAME => 'testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36673/hbase 2024-12-07T01:23:11,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741926_1105 (size=67) 2024-12-07T01:23:11,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741926_1105 (size=67) 2024-12-07T01:23:11,492 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:11,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741926_1105 (size=67) 2024-12-07T01:23:11,493 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,494 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName a 2024-12-07T01:23:11,494 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,495 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,495 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,496 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName b 2024-12-07T01:23:11,496 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,497 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,497 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,498 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName c 2024-12-07T01:23:11,498 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,499 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,499 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,499 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,500 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,501 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,501 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,501 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:23:11,502 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,504 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T01:23:11,505 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9f140ba5e4bd358e794cc587c08a6f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64787035, jitterRate=-0.03459794819355011}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:23:11,505 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9f140ba5e4bd358e794cc587c08a6f0: Writing region info on filesystem at 1733534591492Initializing all the Stores at 1733534591493 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591493Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591493Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591493Cleaning up temporary data from old regions at 1733534591501 (+8 ms)Region opened successfully at 1733534591505 (+4 ms) 2024-12-07T01:23:11,505 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e9f140ba5e4bd358e794cc587c08a6f0, disabling compactions & flushes 2024-12-07T01:23:11,505 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:11,505 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:11,505 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. after waiting 0 ms 2024-12-07T01:23:11,505 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:11,505 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:11,506 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e9f140ba5e4bd358e794cc587c08a6f0: Waiting for close lock at 1733534591505Disabling compacts and flushes for region at 1733534591505Disabling writes for close at 1733534591505Writing region close event to WAL at 1733534591505Closed at 1733534591505 2024-12-07T01:23:11,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741925_1104 (size=93) 2024-12-07T01:23:11,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741925_1104 (size=93) 2024-12-07T01:23:11,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741925_1104 (size=93) 2024-12-07T01:23:11,510 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:23:11,510 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99618403:(num 1733534591466) 2024-12-07T01:23:11,510 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:23:11,512 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:11,526 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512, exclude list is [], retry=0 2024-12-07T01:23:11,529 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:11,529 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:11,529 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:11,531 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512 2024-12-07T01:23:11,531 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991)] 2024-12-07T01:23:11,531 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e9f140ba5e4bd358e794cc587c08a6f0, NAME => 'testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:23:11,531 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:11,531 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,531 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,533 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,533 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName a 2024-12-07T01:23:11,534 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,534 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,534 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,535 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName b 2024-12-07T01:23:11,535 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,536 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,536 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,536 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName c 2024-12-07T01:23:11,536 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,537 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,537 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,538 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,539 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,540 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,540 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,540 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:23:11,542 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,543 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9f140ba5e4bd358e794cc587c08a6f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58883358, jitterRate=-0.12256959080696106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:23:11,543 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9f140ba5e4bd358e794cc587c08a6f0: Writing region info on filesystem at 1733534591532Initializing all the Stores at 1733534591532Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591532Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591533 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591533Cleaning up temporary data from old regions at 1733534591540 (+7 ms)Region opened successfully at 1733534591543 (+3 ms) 2024-12-07T01:23:11,551 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e9f140ba5e4bd358e794cc587c08a6f0 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-07T01:23:11,568 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/a/c5bb4f9e9f374959a5c1a5c1ab3d7a5b is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733534591543/Put/seqid=0 2024-12-07T01:23:11,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741928_1107 (size=5958) 2024-12-07T01:23:11,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741928_1107 (size=5958) 2024-12-07T01:23:11,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741928_1107 (size=5958) 2024-12-07T01:23:11,575 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/a/c5bb4f9e9f374959a5c1a5c1ab3d7a5b 2024-12-07T01:23:11,581 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/a/c5bb4f9e9f374959a5c1a5c1ab3d7a5b as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/a/c5bb4f9e9f374959a5c1a5c1ab3d7a5b 2024-12-07T01:23:11,586 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/a/c5bb4f9e9f374959a5c1a5c1ab3d7a5b, entries=10, sequenceid=13, filesize=5.8 K 2024-12-07T01:23:11,587 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for e9f140ba5e4bd358e794cc587c08a6f0 in 36ms, sequenceid=13, compaction requested=false 2024-12-07T01:23:11,588 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e9f140ba5e4bd358e794cc587c08a6f0: 2024-12-07T01:23:11,604 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e9f140ba5e4bd358e794cc587c08a6f0, disabling compactions & flushes 2024-12-07T01:23:11,604 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:11,604 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:11,604 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. after waiting 0 ms 2024-12-07T01:23:11,604 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:11,605 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:11,605 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:11,605 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e9f140ba5e4bd358e794cc587c08a6f0: Waiting for close lock at 1733534591604Disabling compacts and flushes for region at 1733534591604Disabling writes for close at 1733534591604Writing region close event to WAL at 1733534591605 (+1 ms)Closed at 1733534591605 2024-12-07T01:23:11,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741927_1106 (size=5869) 2024-12-07T01:23:11,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741927_1106 (size=5869) 2024-12-07T01:23:11,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741927_1106 (size=5869) 2024-12-07T01:23:11,620 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512, size=5.7 K (5869bytes) 2024-12-07T01:23:11,620 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512 2024-12-07T01:23:11,621 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512 after 1ms 2024-12-07T01:23:11,622 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512 took 2ms 2024-12-07T01:23:11,624 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512 so closing down 2024-12-07T01:23:11,624 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:23:11,625 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733534591512.temp 2024-12-07T01:23:11,626 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000003-wal.1733534591512.temp 2024-12-07T01:23:11,626 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:23:11,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741929_1108 (size=5223) 2024-12-07T01:23:11,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741929_1108 (size=5223) 2024-12-07T01:23:11,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741929_1108 (size=5223) 2024-12-07T01:23:11,633 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000003-wal.1733534591512.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-07T01:23:11,634 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000003-wal.1733534591512.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000035 2024-12-07T01:23:11,634 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-12-07T01:23:11,634 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512, journal: Splitting hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512, size=5.7 K (5869bytes) at 1733534591620Finishing writing output for hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512 so closing down at 1733534591624 (+4 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000003-wal.1733534591512.temp at 1733534591626 (+2 ms)3 split writer threads finished at 1733534591626Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000003-wal.1733534591512.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733534591633 (+7 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000003-wal.1733534591512.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000035 at 1733534591634 (+1 ms)Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1733534591634 2024-12-07T01:23:11,636 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591512 to hdfs://localhost:36673/hbase/oldWALs/wal.1733534591512 2024-12-07T01:23:11,637 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000035 2024-12-07T01:23:11,637 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:23:11,638 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:11,650 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639, exclude list is [], retry=0 2024-12-07T01:23:11,653 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:11,653 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:11,653 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:11,654 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 2024-12-07T01:23:11,654 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:34991:34991)] 2024-12-07T01:23:11,655 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e9f140ba5e4bd358e794cc587c08a6f0, NAME => 'testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0.', STARTKEY => '', ENDKEY => ''} 2024-12-07T01:23:11,655 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:11,655 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,655 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,656 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,657 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName a 2024-12-07T01:23:11,657 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,662 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/a/c5bb4f9e9f374959a5c1a5c1ab3d7a5b 2024-12-07T01:23:11,662 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,662 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,663 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName b 2024-12-07T01:23:11,663 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,664 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,664 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,665 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName c 2024-12-07T01:23:11,665 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:11,666 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:11,666 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,666 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,668 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,669 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000035 2024-12-07T01:23:11,672 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000035 2024-12-07T01:23:11,672 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e9f140ba5e4bd358e794cc587c08a6f0 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-07T01:23:11,688 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/b/a6e5628e6b104a5c854e1f58d9e37f15 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733534591588/Put/seqid=0 2024-12-07T01:23:11,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741931_1110 (size=5958) 2024-12-07T01:23:11,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741931_1110 (size=5958) 2024-12-07T01:23:11,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741931_1110 (size=5958) 2024-12-07T01:23:11,695 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/b/a6e5628e6b104a5c854e1f58d9e37f15 2024-12-07T01:23:11,711 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/c/f0823647a4d744d799a9c45b0a251bab is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733534591595/Put/seqid=0 2024-12-07T01:23:11,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741932_1111 (size=5958) 2024-12-07T01:23:11,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741932_1111 (size=5958) 2024-12-07T01:23:11,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741932_1111 (size=5958) 2024-12-07T01:23:11,718 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/c/f0823647a4d744d799a9c45b0a251bab 2024-12-07T01:23:11,722 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/b/a6e5628e6b104a5c854e1f58d9e37f15 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/b/a6e5628e6b104a5c854e1f58d9e37f15 2024-12-07T01:23:11,727 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/b/a6e5628e6b104a5c854e1f58d9e37f15, entries=10, sequenceid=35, filesize=5.8 K 2024-12-07T01:23:11,728 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/c/f0823647a4d744d799a9c45b0a251bab as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/c/f0823647a4d744d799a9c45b0a251bab 2024-12-07T01:23:11,732 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/c/f0823647a4d744d799a9c45b0a251bab, entries=10, sequenceid=35, filesize=5.8 K 2024-12-07T01:23:11,732 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for e9f140ba5e4bd358e794cc587c08a6f0 in 60ms, sequenceid=35, compaction requested=false; wal=null 2024-12-07T01:23:11,733 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000035 2024-12-07T01:23:11,734 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,734 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,734 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:23:11,736 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:11,738 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-07T01:23:11,738 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9f140ba5e4bd358e794cc587c08a6f0; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62095313, jitterRate=-0.07470773160457611}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:23:11,739 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9f140ba5e4bd358e794cc587c08a6f0: Writing region info on filesystem at 1733534591655Initializing all the Stores at 1733534591656 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591656Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591656Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534591656Obtaining lock to block concurrent updates at 1733534591672 (+16 ms)Preparing flush snapshotting stores in e9f140ba5e4bd358e794cc587c08a6f0 at 1733534591672Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733534591673 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. at 1733534591673Flushing e9f140ba5e4bd358e794cc587c08a6f0/b: creating writer at 1733534591673Flushing e9f140ba5e4bd358e794cc587c08a6f0/b: appending metadata at 1733534591688 (+15 ms)Flushing e9f140ba5e4bd358e794cc587c08a6f0/b: closing flushed file at 1733534591688Flushing e9f140ba5e4bd358e794cc587c08a6f0/c: creating writer at 1733534591699 (+11 ms)Flushing e9f140ba5e4bd358e794cc587c08a6f0/c: appending metadata at 1733534591711 (+12 ms)Flushing e9f140ba5e4bd358e794cc587c08a6f0/c: closing flushed file at 1733534591711Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@443ed5f2: reopening flushed file at 1733534591722 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6159d80f: reopening flushed file at 1733534591727 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for e9f140ba5e4bd358e794cc587c08a6f0 in 60ms, sequenceid=35, compaction requested=false; wal=null at 1733534591732 (+5 ms)Cleaning up temporary data from old regions at 1733534591734 (+2 ms)Region opened successfully at 1733534591738 (+4 ms) 2024-12-07T01:23:11,762 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T01:23:11,800 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639, size=0 (0bytes) 2024-12-07T01:23:11,800 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 might be still open, length is 0 2024-12-07T01:23:11,800 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 2024-12-07T01:23:11,800 WARN [IPC Server handler 3 on default port 36673 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-07T01:23:11,800 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 after 0ms 2024-12-07T01:23:12,503 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:43036 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45293:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43036 dst: /127.0.0.1:45293 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45293 remote=/127.0.0.1:43036]. Total timeout mills is 60000, 59261 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:12,505 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:33150 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:46343:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33150 dst: /127.0.0.1:46343 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:12,504 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-70147621_22 at /127.0.0.1:47322 [Receiving block BP-773003372-172.17.0.3-1733534552230:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33463:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47322 dst: /127.0.0.1:33463 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:12,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741930_1112 (size=5215) 2024-12-07T01:23:12,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741930_1112 (size=5215) 2024-12-07T01:23:12,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741930_1112 (size=5215) 2024-12-07T01:23:15,802 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 after 4002ms 2024-12-07T01:23:15,808 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 took 4009ms 2024-12-07T01:23:15,812 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639; continuing. 2024-12-07T01:23:15,812 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 so closing down 2024-12-07T01:23:15,812 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-07T01:23:15,815 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733534591639.temp 2024-12-07T01:23:15,817 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000037-wal.1733534591639.temp 2024-12-07T01:23:15,817 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-07T01:23:15,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741933_1113 (size=5223) 2024-12-07T01:23:15,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741933_1113 (size=5223) 2024-12-07T01:23:15,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741933_1113 (size=5223) 2024-12-07T01:23:15,826 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000037-wal.1733534591639.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-07T01:23:15,828 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000037-wal.1733534591639.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000066 2024-12-07T01:23:15,828 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639, size=0, length=0, corrupted=false, cancelled=false 2024-12-07T01:23:15,828 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639, journal: Splitting hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639, size=0 (0bytes) at 1733534591800Finishing writing output for hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 so closing down at 1733534595812 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000037-wal.1733534591639.temp at 1733534595817 (+5 ms)3 split writer threads finished at 1733534595818 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000037-wal.1733534591639.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733534595826 (+8 ms)Rename recovered edits hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000037-wal.1733534591639.temp to hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000066 at 1733534595828 (+2 ms)Processed 30 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639, size=0, length=0, corrupted=false, cancelled=false at 1733534595828 2024-12-07T01:23:15,830 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 to hdfs://localhost:36673/hbase/oldWALs/wal.1733534591639 2024-12-07T01:23:15,832 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000066 2024-12-07T01:23:15,832 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-07T01:23:15,834 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36673/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461, archiveDir=hdfs://localhost:36673/hbase/oldWALs, maxLogs=32 2024-12-07T01:23:15,849 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534595834, exclude list is [], retry=0 2024-12-07T01:23:15,851 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33463,DS-69389200-bc4b-4ba0-81f8-76b097d98bbd,DISK] 2024-12-07T01:23:15,851 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45293,DS-4f4d16f8-618a-419f-9a6f-ab987e9c15a7,DISK] 2024-12-07T01:23:15,852 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46343,DS-ef9cc900-aef9-4a1e-8b06-9f5b5c5d1f63,DISK] 2024-12-07T01:23:15,853 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534595834 2024-12-07T01:23:15,853 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44753:44753),(127.0.0.1/127.0.0.1:33627:33627),(127.0.0.1/127.0.0.1:34991:34991)] 2024-12-07T01:23:15,854 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T01:23:15,855 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:15,855 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName a 2024-12-07T01:23:15,856 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:15,860 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/a/c5bb4f9e9f374959a5c1a5c1ab3d7a5b 2024-12-07T01:23:15,861 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:15,861 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:15,861 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName b 2024-12-07T01:23:15,861 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:15,866 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/b/a6e5628e6b104a5c854e1f58d9e37f15 2024-12-07T01:23:15,866 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:15,866 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:15,867 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f140ba5e4bd358e794cc587c08a6f0 columnFamilyName c 2024-12-07T01:23:15,867 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T01:23:15,873 DEBUG [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/c/f0823647a4d744d799a9c45b0a251bab 2024-12-07T01:23:15,873 INFO [StoreOpener-e9f140ba5e4bd358e794cc587c08a6f0-1 {}] regionserver.HStore(327): Store=e9f140ba5e4bd358e794cc587c08a6f0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T01:23:15,873 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:15,874 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:15,875 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:15,876 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000066 2024-12-07T01:23:15,882 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000066 2024-12-07T01:23:15,882 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e9f140ba5e4bd358e794cc587c08a6f0 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-07T01:23:15,895 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/a/7fe5218cedc74f75b3307b40895142b9 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733534591744/Put/seqid=0 2024-12-07T01:23:15,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741935_1115 (size=5958) 2024-12-07T01:23:15,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741935_1115 (size=5958) 2024-12-07T01:23:15,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741935_1115 (size=5958) 2024-12-07T01:23:15,901 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/a/7fe5218cedc74f75b3307b40895142b9 2024-12-07T01:23:15,918 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/b/739c8c51877a4bb1bd5ce01a7c37a628 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733534591751/Put/seqid=0 2024-12-07T01:23:15,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741936_1116 (size=5958) 2024-12-07T01:23:15,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741936_1116 (size=5958) 2024-12-07T01:23:15,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741936_1116 (size=5958) 2024-12-07T01:23:15,925 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/b/739c8c51877a4bb1bd5ce01a7c37a628 2024-12-07T01:23:15,942 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/c/654ee567960741d1b5e4718c9deaf2c6 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733534591757/Put/seqid=0 2024-12-07T01:23:15,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741937_1117 (size=5958) 2024-12-07T01:23:15,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741937_1117 (size=5958) 2024-12-07T01:23:15,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741937_1117 (size=5958) 2024-12-07T01:23:15,949 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/c/654ee567960741d1b5e4718c9deaf2c6 2024-12-07T01:23:15,953 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/a/7fe5218cedc74f75b3307b40895142b9 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/a/7fe5218cedc74f75b3307b40895142b9 2024-12-07T01:23:15,958 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/a/7fe5218cedc74f75b3307b40895142b9, entries=10, sequenceid=66, filesize=5.8 K 2024-12-07T01:23:15,959 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/b/739c8c51877a4bb1bd5ce01a7c37a628 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/b/739c8c51877a4bb1bd5ce01a7c37a628 2024-12-07T01:23:15,963 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/b/739c8c51877a4bb1bd5ce01a7c37a628, entries=10, sequenceid=66, filesize=5.8 K 2024-12-07T01:23:15,965 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/.tmp/c/654ee567960741d1b5e4718c9deaf2c6 as hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/c/654ee567960741d1b5e4718c9deaf2c6 2024-12-07T01:23:15,969 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/c/654ee567960741d1b5e4718c9deaf2c6, entries=10, sequenceid=66, filesize=5.8 K 2024-12-07T01:23:15,970 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for e9f140ba5e4bd358e794cc587c08a6f0 in 87ms, sequenceid=66, compaction requested=false; wal=null 2024-12-07T01:23:15,970 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/0000000000000000066 2024-12-07T01:23:15,971 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:15,971 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:15,972 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-07T01:23:15,973 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9f140ba5e4bd358e794cc587c08a6f0 2024-12-07T01:23:15,975 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/hbase/data/default/testReplayEditsWrittenViaHRegion/e9f140ba5e4bd358e794cc587c08a6f0/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-07T01:23:15,976 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9f140ba5e4bd358e794cc587c08a6f0; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67329150, jitterRate=0.003282517194747925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-07T01:23:15,977 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9f140ba5e4bd358e794cc587c08a6f0: Writing region info on filesystem at 1733534595854Initializing all the Stores at 1733534595855 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534595855Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534595855Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733534595855Obtaining lock to block concurrent updates at 1733534595882 (+27 ms)Preparing flush snapshotting stores in e9f140ba5e4bd358e794cc587c08a6f0 at 1733534595882Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733534595882Flushing stores of testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. at 1733534595882Flushing e9f140ba5e4bd358e794cc587c08a6f0/a: creating writer at 1733534595882Flushing e9f140ba5e4bd358e794cc587c08a6f0/a: appending metadata at 1733534595894 (+12 ms)Flushing e9f140ba5e4bd358e794cc587c08a6f0/a: closing flushed file at 1733534595894Flushing e9f140ba5e4bd358e794cc587c08a6f0/b: creating writer at 1733534595905 (+11 ms)Flushing e9f140ba5e4bd358e794cc587c08a6f0/b: appending metadata at 1733534595918 (+13 ms)Flushing e9f140ba5e4bd358e794cc587c08a6f0/b: closing flushed file at 1733534595918Flushing e9f140ba5e4bd358e794cc587c08a6f0/c: creating writer at 1733534595929 (+11 ms)Flushing e9f140ba5e4bd358e794cc587c08a6f0/c: appending metadata at 1733534595941 (+12 ms)Flushing e9f140ba5e4bd358e794cc587c08a6f0/c: closing flushed file at 1733534595941Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e56dbe8: reopening flushed file at 1733534595952 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12633c92: reopening flushed file at 1733534595958 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4557cc00: reopening flushed file at 1733534595964 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for e9f140ba5e4bd358e794cc587c08a6f0 in 87ms, sequenceid=66, compaction requested=false; wal=null at 1733534595970 (+6 ms)Cleaning up temporary data from old regions at 1733534595971 (+1 ms)Region opened successfully at 1733534595976 (+5 ms) 2024-12-07T01:23:15,988 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e9f140ba5e4bd358e794cc587c08a6f0, disabling compactions & flushes 2024-12-07T01:23:15,988 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:15,988 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:15,988 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. after waiting 0 ms 2024-12-07T01:23:15,988 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:15,989 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733534591462.e9f140ba5e4bd358e794cc587c08a6f0. 2024-12-07T01:23:15,990 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e9f140ba5e4bd358e794cc587c08a6f0: Waiting for close lock at 1733534595988Disabling compacts and flushes for region at 1733534595988Disabling writes for close at 1733534595988Writing region close event to WAL at 1733534595989 (+1 ms)Closed at 1733534595989 2024-12-07T01:23:15,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741934_1114 (size=93) 2024-12-07T01:23:15,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741934_1114 (size=93) 2024-12-07T01:23:15,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741934_1114 (size=93) 2024-12-07T01:23:15,994 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-07T01:23:15,994 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733534595834) 2024-12-07T01:23:16,009 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=450 (was 443) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:36673 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-653677585_22 at /127.0.0.1:58096 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:36673 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-653677585_22 at /127.0.0.1:47776 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-653677585_22 at /127.0.0.1:37982 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1347) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 240), ProcessCount=11 (was 11), AvailableMemoryMB=4824 (was 4843) 2024-12-07T01:23:16,009 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-07T01:23:16,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T01:23:16,009 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T01:23:16,009 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T01:23:16,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:23:16,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:23:16,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T01:23:16,010 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T01:23:16,010 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=805536183, stopped=false 2024-12-07T01:23:16,010 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ec1863dc21e5,45579,1733534557376 2024-12-07T01:23:16,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T01:23:16,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T01:23:16,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T01:23:16,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:23:16,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:23:16,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:23:16,070 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:23:16,071 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:23:16,071 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:23:16,071 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T01:23:16,072 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T01:23:16,072 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T01:23:16,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:23:16,072 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec1863dc21e5,35781,1733534558145' ***** 2024-12-07T01:23:16,072 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T01:23:16,072 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec1863dc21e5,35853,1733534558237' ***** 2024-12-07T01:23:16,072 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T01:23:16,073 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T01:23:16,073 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T01:23:16,073 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T01:23:16,073 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T01:23:16,073 INFO [RS:0;ec1863dc21e5:35781 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T01:23:16,073 INFO [RS:1;ec1863dc21e5:35853 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T01:23:16,073 INFO [RS:1;ec1863dc21e5:35853 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T01:23:16,073 INFO [RS:0;ec1863dc21e5:35781 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T01:23:16,073 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(3091): Received CLOSE for 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:23:16,073 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(959): stopping server ec1863dc21e5,35781,1733534558145 2024-12-07T01:23:16,073 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T01:23:16,073 INFO [RS:0;ec1863dc21e5:35781 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ec1863dc21e5:35781. 2024-12-07T01:23:16,073 DEBUG [RS:0;ec1863dc21e5:35781 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T01:23:16,073 DEBUG [RS:0;ec1863dc21e5:35781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:23:16,073 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(976): stopping server ec1863dc21e5,35781,1733534558145; all regions closed. 2024-12-07T01:23:16,073 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(959): stopping server ec1863dc21e5,35853,1733534558237 2024-12-07T01:23:16,073 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T01:23:16,073 INFO [RS:1;ec1863dc21e5:35853 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ec1863dc21e5:35853. 2024-12-07T01:23:16,074 DEBUG [RS:1;ec1863dc21e5:35853 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T01:23:16,074 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3848f991c723c9b72b68e9365062aeee, disabling compactions & flushes 2024-12-07T01:23:16,074 DEBUG [RS:1;ec1863dc21e5:35853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:23:16,074 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:23:16,074 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T01:23:16,074 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:23:16,074 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T01:23:16,074 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T01:23:16,074 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. after waiting 0 ms 2024-12-07T01:23:16,074 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:23:16,074 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T01:23:16,074 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-07T01:23:16,074 DEBUG [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 3848f991c723c9b72b68e9365062aeee=testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee.} 2024-12-07T01:23:16,074 DEBUG [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3848f991c723c9b72b68e9365062aeee 2024-12-07T01:23:16,074 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T01:23:16,074 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T01:23:16,075 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T01:23:16,075 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T01:23:16,075 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T01:23:16,075 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-07T01:23:16,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741835_1011 (size=2598) 2024-12-07T01:23:16,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741835_1011 (size=2598) 2024-12-07T01:23:16,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741835_1011 (size=2598) 2024-12-07T01:23:16,079 DEBUG [RS:0;ec1863dc21e5:35781 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/oldWALs 2024-12-07T01:23:16,080 INFO [RS:0;ec1863dc21e5:35781 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ec1863dc21e5%2C35781%2C1733534558145:(num 1733534559752) 2024-12-07T01:23:16,080 DEBUG [RS:0;ec1863dc21e5:35781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:23:16,080 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:23:16,080 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T01:23:16,080 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.ChoreService(370): Chore service for: regionserver/ec1863dc21e5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-07T01:23:16,080 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T01:23:16,080 INFO [regionserver/ec1863dc21e5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T01:23:16,080 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T01:23:16,080 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T01:23:16,080 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T01:23:16,080 INFO [RS:0;ec1863dc21e5:35781 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35781 2024-12-07T01:23:16,080 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3848f991c723c9b72b68e9365062aeee/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-07T01:23:16,082 INFO [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:23:16,082 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3848f991c723c9b72b68e9365062aeee: Waiting for close lock at 1733534596073Running coprocessor pre-close hooks at 1733534596073Disabling compacts and flushes for region at 1733534596073Disabling writes for close at 1733534596074 (+1 ms)Writing region close event to WAL at 1733534596076 (+2 ms)Running coprocessor post-close hooks at 1733534596082 (+6 ms)Closed at 1733534596082 2024-12-07T01:23:16,082 DEBUG [RS_CLOSE_REGION-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee. 2024-12-07T01:23:16,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec1863dc21e5,35781,1733534558145 2024-12-07T01:23:16,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T01:23:16,091 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T01:23:16,097 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/.tmp/info/01624a8e838149a791969bf1c82c8d4b is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733534572750.3848f991c723c9b72b68e9365062aeee./info:regioninfo/1733534575886/Put/seqid=0 2024-12-07T01:23:16,102 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec1863dc21e5,35781,1733534558145] 2024-12-07T01:23:16,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741938_1118 (size=8243) 2024-12-07T01:23:16,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741938_1118 (size=8243) 2024-12-07T01:23:16,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741938_1118 (size=8243) 2024-12-07T01:23:16,104 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/.tmp/info/01624a8e838149a791969bf1c82c8d4b 2024-12-07T01:23:16,112 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec1863dc21e5,35781,1733534558145 already deleted, retry=false 2024-12-07T01:23:16,112 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec1863dc21e5,35781,1733534558145 expired; onlineServers=1 2024-12-07T01:23:16,129 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/.tmp/ns/2327624d6d2e4ab183401c32a92e0b3e is 43, key is default/ns:d/1733534560561/Put/seqid=0 2024-12-07T01:23:16,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741939_1119 (size=5153) 2024-12-07T01:23:16,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741939_1119 (size=5153) 2024-12-07T01:23:16,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741939_1119 (size=5153) 2024-12-07T01:23:16,137 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/.tmp/ns/2327624d6d2e4ab183401c32a92e0b3e 2024-12-07T01:23:16,151 INFO [regionserver/ec1863dc21e5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:23:16,151 INFO [regionserver/ec1863dc21e5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:23:16,158 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/.tmp/table/0bdd87d8c02a4af38498f21e67ed0c8d is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733534573173/Put/seqid=0 2024-12-07T01:23:16,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741940_1120 (size=5431) 2024-12-07T01:23:16,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741940_1120 (size=5431) 2024-12-07T01:23:16,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741940_1120 (size=5431) 2024-12-07T01:23:16,165 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/.tmp/table/0bdd87d8c02a4af38498f21e67ed0c8d 2024-12-07T01:23:16,171 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/.tmp/info/01624a8e838149a791969bf1c82c8d4b as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/info/01624a8e838149a791969bf1c82c8d4b 2024-12-07T01:23:16,177 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/info/01624a8e838149a791969bf1c82c8d4b, entries=18, sequenceid=21, filesize=8.0 K 2024-12-07T01:23:16,179 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/.tmp/ns/2327624d6d2e4ab183401c32a92e0b3e as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/ns/2327624d6d2e4ab183401c32a92e0b3e 2024-12-07T01:23:16,184 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/ns/2327624d6d2e4ab183401c32a92e0b3e, entries=2, sequenceid=21, filesize=5.0 K 2024-12-07T01:23:16,185 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/.tmp/table/0bdd87d8c02a4af38498f21e67ed0c8d as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/table/0bdd87d8c02a4af38498f21e67ed0c8d 2024-12-07T01:23:16,191 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/table/0bdd87d8c02a4af38498f21e67ed0c8d, entries=2, sequenceid=21, filesize=5.3 K 2024-12-07T01:23:16,192 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 117ms, sequenceid=21, compaction requested=false 2024-12-07T01:23:16,196 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-07T01:23:16,197 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T01:23:16,197 INFO [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T01:23:16,197 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733534596074Running coprocessor pre-close hooks at 1733534596074Disabling compacts and flushes for region at 1733534596074Disabling writes for close at 1733534596075 (+1 ms)Obtaining lock to block concurrent updates at 1733534596075Preparing flush snapshotting stores in 1588230740 at 1733534596075Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733534596075Flushing stores of hbase:meta,,1.1588230740 at 1733534596077 (+2 ms)Flushing 1588230740/info: creating writer at 1733534596077Flushing 1588230740/info: appending metadata at 1733534596096 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733534596096Flushing 1588230740/ns: creating writer at 1733534596111 (+15 ms)Flushing 1588230740/ns: appending metadata at 1733534596129 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733534596129Flushing 1588230740/table: creating writer at 1733534596144 (+15 ms)Flushing 1588230740/table: appending metadata at 1733534596158 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733534596158Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c9fc842: reopening flushed file at 1733534596170 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53af534f: reopening flushed file at 1733534596178 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14d77a91: reopening flushed file at 1733534596184 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 117ms, sequenceid=21, compaction requested=false at 1733534596192 (+8 ms)Writing region close event to WAL at 1733534596193 (+1 ms)Running coprocessor post-close hooks at 1733534596197 (+4 ms)Closed at 1733534596197 2024-12-07T01:23:16,197 DEBUG [RS_CLOSE_META-regionserver/ec1863dc21e5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T01:23:16,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:23:16,202 INFO [RS:0;ec1863dc21e5:35781 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T01:23:16,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35781-0x101ad621d6e0001, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:23:16,202 INFO [RS:0;ec1863dc21e5:35781 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec1863dc21e5,35781,1733534558145; zookeeper connection closed. 2024-12-07T01:23:16,202 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c1b42b5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c1b42b5 2024-12-07T01:23:16,275 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(976): stopping server ec1863dc21e5,35853,1733534558237; all regions closed. 2024-12-07T01:23:16,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741836_1012 (size=8899) 2024-12-07T01:23:16,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741836_1012 (size=8899) 2024-12-07T01:23:16,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741836_1012 (size=8899) 2024-12-07T01:23:16,282 DEBUG [RS:1;ec1863dc21e5:35853 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/oldWALs 2024-12-07T01:23:16,282 INFO [RS:1;ec1863dc21e5:35853 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ec1863dc21e5%2C35853%2C1733534558237.meta:.meta(num 1733534560393) 2024-12-07T01:23:16,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741833_1009 (size=928) 2024-12-07T01:23:16,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741833_1009 (size=928) 2024-12-07T01:23:16,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741833_1009 (size=928) 2024-12-07T01:23:16,289 DEBUG [RS:1;ec1863dc21e5:35853 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/oldWALs 2024-12-07T01:23:16,289 INFO [RS:1;ec1863dc21e5:35853 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ec1863dc21e5%2C35853%2C1733534558237:(num 1733534559752) 2024-12-07T01:23:16,289 DEBUG [RS:1;ec1863dc21e5:35853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T01:23:16,289 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T01:23:16,289 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T01:23:16,289 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.ChoreService(370): Chore service for: regionserver/ec1863dc21e5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T01:23:16,289 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T01:23:16,289 INFO [regionserver/ec1863dc21e5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T01:23:16,289 INFO [RS:1;ec1863dc21e5:35853 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35853 2024-12-07T01:23:16,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec1863dc21e5,35853,1733534558237 2024-12-07T01:23:16,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T01:23:16,302 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T01:23:16,312 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec1863dc21e5,35853,1733534558237] 2024-12-07T01:23:16,323 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec1863dc21e5,35853,1733534558237 already deleted, retry=false 2024-12-07T01:23:16,323 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec1863dc21e5,35853,1733534558237 expired; onlineServers=0 2024-12-07T01:23:16,323 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ec1863dc21e5,45579,1733534557376' ***** 2024-12-07T01:23:16,323 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T01:23:16,323 INFO [M:0;ec1863dc21e5:45579 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T01:23:16,323 INFO [M:0;ec1863dc21e5:45579 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T01:23:16,323 DEBUG [M:0;ec1863dc21e5:45579 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T01:23:16,323 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T01:23:16,324 DEBUG [M:0;ec1863dc21e5:45579 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T01:23:16,323 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.large.0-1733534559377 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.large.0-1733534559377,5,FailOnTimeoutGroup] 2024-12-07T01:23:16,323 DEBUG [master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.small.0-1733534559378 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec1863dc21e5:0:becomeActiveMaster-HFileCleaner.small.0-1733534559378,5,FailOnTimeoutGroup] 2024-12-07T01:23:16,324 INFO [M:0;ec1863dc21e5:45579 {}] hbase.ChoreService(370): Chore service for: master/ec1863dc21e5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T01:23:16,324 INFO [M:0;ec1863dc21e5:45579 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T01:23:16,324 DEBUG [M:0;ec1863dc21e5:45579 {}] master.HMaster(1795): Stopping service threads 2024-12-07T01:23:16,324 INFO [M:0;ec1863dc21e5:45579 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T01:23:16,325 INFO [M:0;ec1863dc21e5:45579 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T01:23:16,325 INFO [M:0;ec1863dc21e5:45579 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T01:23:16,326 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T01:23:16,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T01:23:16,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T01:23:16,334 DEBUG [M:0;ec1863dc21e5:45579 {}] zookeeper.ZKUtil(347): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T01:23:16,334 WARN [M:0;ec1863dc21e5:45579 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T01:23:16,335 INFO [M:0;ec1863dc21e5:45579 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/.lastflushedseqids 2024-12-07T01:23:16,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741941_1121 (size=138) 2024-12-07T01:23:16,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741941_1121 (size=138) 2024-12-07T01:23:16,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741941_1121 (size=138) 2024-12-07T01:23:16,354 INFO [M:0;ec1863dc21e5:45579 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T01:23:16,354 INFO [M:0;ec1863dc21e5:45579 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T01:23:16,355 DEBUG [M:0;ec1863dc21e5:45579 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T01:23:16,355 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:23:16,355 DEBUG [M:0;ec1863dc21e5:45579 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:23:16,355 DEBUG [M:0;ec1863dc21e5:45579 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T01:23:16,355 DEBUG [M:0;ec1863dc21e5:45579 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:23:16,355 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.33 KB heapSize=83.72 KB 2024-12-07T01:23:16,369 DEBUG [M:0;ec1863dc21e5:45579 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/946d6d049ad94721b331a9deeae8641b is 82, key is hbase:meta,,1/info:regioninfo/1733534560480/Put/seqid=0 2024-12-07T01:23:16,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741942_1122 (size=5672) 2024-12-07T01:23:16,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741942_1122 (size=5672) 2024-12-07T01:23:16,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741942_1122 (size=5672) 2024-12-07T01:23:16,375 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/946d6d049ad94721b331a9deeae8641b 2024-12-07T01:23:16,393 DEBUG [M:0;ec1863dc21e5:45579 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/68a7e4a694f84621a51cbcf787521967 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733534573181/Put/seqid=0 2024-12-07T01:23:16,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741943_1123 (size=7754) 2024-12-07T01:23:16,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741943_1123 (size=7754) 2024-12-07T01:23:16,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741943_1123 (size=7754) 2024-12-07T01:23:16,403 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/68a7e4a694f84621a51cbcf787521967 2024-12-07T01:23:16,407 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 68a7e4a694f84621a51cbcf787521967 2024-12-07T01:23:16,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:23:16,412 INFO [RS:1;ec1863dc21e5:35853 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T01:23:16,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35853-0x101ad621d6e0002, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:23:16,412 INFO [RS:1;ec1863dc21e5:35853 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec1863dc21e5,35853,1733534558237; zookeeper connection closed. 2024-12-07T01:23:16,413 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5d907591 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5d907591 2024-12-07T01:23:16,413 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-07T01:23:16,421 DEBUG [M:0;ec1863dc21e5:45579 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a3640b1155547b5ac57b878b879a5a8 is 69, key is ec1863dc21e5,35781,1733534558145/rs:state/1733534559450/Put/seqid=0 2024-12-07T01:23:16,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741944_1124 (size=5440) 2024-12-07T01:23:16,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741944_1124 (size=5440) 2024-12-07T01:23:16,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741944_1124 (size=5440) 2024-12-07T01:23:16,428 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a3640b1155547b5ac57b878b879a5a8 2024-12-07T01:23:16,431 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2a3640b1155547b5ac57b878b879a5a8 2024-12-07T01:23:16,432 DEBUG [M:0;ec1863dc21e5:45579 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/946d6d049ad94721b331a9deeae8641b as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/946d6d049ad94721b331a9deeae8641b 2024-12-07T01:23:16,436 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/946d6d049ad94721b331a9deeae8641b, entries=8, sequenceid=168, filesize=5.5 K 2024-12-07T01:23:16,437 DEBUG [M:0;ec1863dc21e5:45579 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/68a7e4a694f84621a51cbcf787521967 as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/68a7e4a694f84621a51cbcf787521967 2024-12-07T01:23:16,441 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 68a7e4a694f84621a51cbcf787521967 2024-12-07T01:23:16,441 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/68a7e4a694f84621a51cbcf787521967, entries=17, sequenceid=168, filesize=7.6 K 2024-12-07T01:23:16,442 DEBUG [M:0;ec1863dc21e5:45579 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a3640b1155547b5ac57b878b879a5a8 as hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a3640b1155547b5ac57b878b879a5a8 2024-12-07T01:23:16,446 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2a3640b1155547b5ac57b878b879a5a8 2024-12-07T01:23:16,447 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36673/user/jenkins/test-data/26dd7db4-c33a-5552-39af-e3b11594d9fe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a3640b1155547b5ac57b878b879a5a8, entries=3, sequenceid=168, filesize=5.3 K 2024-12-07T01:23:16,448 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=168, compaction requested=false 2024-12-07T01:23:16,449 INFO [M:0;ec1863dc21e5:45579 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T01:23:16,449 DEBUG [M:0;ec1863dc21e5:45579 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733534596355Disabling compacts and flushes for region at 1733534596355Disabling writes for close at 1733534596355Obtaining lock to block concurrent updates at 1733534596355Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733534596355Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69972, getHeapSize=85664, getOffHeapSize=0, getCellsCount=195 at 1733534596355Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733534596356 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733534596356Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733534596369 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733534596369Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733534596379 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733534596392 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733534596392Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733534596408 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733534596421 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733534596421Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ff7151f: reopening flushed file at 1733534596431 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68225d0a: reopening flushed file at 1733534596436 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6da36c58: reopening flushed file at 1733534596442 (+6 ms)Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=168, compaction requested=false at 1733534596448 (+6 ms)Writing region close event to WAL at 1733534596449 (+1 ms)Closed at 1733534596449 2024-12-07T01:23:16,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45293 is added to blk_1073741830_1006 (size=81419) 2024-12-07T01:23:16,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46343 is added to blk_1073741830_1006 (size=81419) 2024-12-07T01:23:16,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33463 is added to blk_1073741830_1006 (size=81419) 2024-12-07T01:23:16,452 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T01:23:16,452 INFO [M:0;ec1863dc21e5:45579 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T01:23:16,452 INFO [M:0;ec1863dc21e5:45579 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45579 2024-12-07T01:23:16,452 INFO [M:0;ec1863dc21e5:45579 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T01:23:16,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:23:16,560 INFO [M:0;ec1863dc21e5:45579 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T01:23:16,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45579-0x101ad621d6e0000, quorum=127.0.0.1:61145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T01:23:16,570 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576939 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576939 (inode 16655) Holder DFSClient_NONMAPREDUCE_-70147621_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733534576181/wal.1733534576939 (inode 16655) Holder DFSClient_NONMAPREDUCE_-70147621_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-07T01:23:16,571 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733534567919/wal.1733534568019 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:16,571 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733534591461/wal.1733534591639 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:16,574 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534586282 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534586282 (inode 16678) Holder DFSClient_NONMAPREDUCE_-70147621_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733534577068/wal.1733534586282 (inode 16678) Holder DFSClient_NONMAPREDUCE_-70147621_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-07T01:23:16,574 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733534561618/wal.1733534561684 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:16,576 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586900 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586900 (inode 16704) Holder DFSClient_NONMAPREDUCE_-70147621_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733534586480/wal.1733534586900 (inode 16704) Holder DFSClient_NONMAPREDUCE_-70147621_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-07T01:23:16,577 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733534587043/wal.1733534587119 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T01:23:16,579 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal.1733534561498 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal.1733534561498 (inode 16485) Holder DFSClient_NONMAPREDUCE_-70147621_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733534561327/wal.1733534561498 (inode 16485) Holder DFSClient_NONMAPREDUCE_-70147621_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-07T01:23:16,581 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal.1733534561093 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal.1733534561093 (inode 16462) Holder DFSClient_NONMAPREDUCE_-70147621_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733534560849/wal.1733534561093 (inode 16462) Holder DFSClient_NONMAPREDUCE_-70147621_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-07T01:23:16,586 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:23:16,588 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T01:23:16,588 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T01:23:16,588 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T01:23:16,588 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/hadoop.log.dir/,STOPPED} 2024-12-07T01:23:16,613 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-07T01:23:16,613 WARN [BP-773003372-172.17.0.3-1733534552230 heartbeating to localhost/127.0.0.1:36673 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T01:23:16,613 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T01:23:16,613 WARN [BP-773003372-172.17.0.3-1733534552230 heartbeating to localhost/127.0.0.1:36673 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-773003372-172.17.0.3-1733534552230 (Datanode Uuid 165fafa2-1111-4fc3-aa94-022ea53b0d11) service to localhost/127.0.0.1:36673 2024-12-07T01:23:16,614 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data5/current/BP-773003372-172.17.0.3-1733534552230 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:23:16,614 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data6/current/BP-773003372-172.17.0.3-1733534552230 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:23:16,615 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T01:23:16,617 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:23:16,617 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T01:23:16,617 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T01:23:16,617 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T01:23:16,617 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/hadoop.log.dir/,STOPPED} 2024-12-07T01:23:16,618 WARN [BP-773003372-172.17.0.3-1733534552230 heartbeating to localhost/127.0.0.1:36673 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T01:23:16,618 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-07T01:23:16,618 WARN [BP-773003372-172.17.0.3-1733534552230 heartbeating to localhost/127.0.0.1:36673 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-773003372-172.17.0.3-1733534552230 (Datanode Uuid ae426d4e-915d-48c1-af82-321a9dd4dd4c) service to localhost/127.0.0.1:36673 2024-12-07T01:23:16,618 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T01:23:16,619 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data3/current/BP-773003372-172.17.0.3-1733534552230 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:23:16,619 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data4/current/BP-773003372-172.17.0.3-1733534552230 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:23:16,619 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T01:23:16,621 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T01:23:16,621 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T01:23:16,621 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T01:23:16,621 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T01:23:16,621 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/hadoop.log.dir/,STOPPED} 2024-12-07T01:23:16,623 WARN [BP-773003372-172.17.0.3-1733534552230 heartbeating to localhost/127.0.0.1:36673 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T01:23:16,623 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-07T01:23:16,623 WARN [BP-773003372-172.17.0.3-1733534552230 heartbeating to localhost/127.0.0.1:36673 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-773003372-172.17.0.3-1733534552230 (Datanode Uuid a3c8590f-98f1-416a-9096-404b5ea22c8d) service to localhost/127.0.0.1:36673 2024-12-07T01:23:16,623 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T01:23:16,624 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data1/current/BP-773003372-172.17.0.3-1733534552230 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:23:16,624 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/cluster_0ed2c8dd-350c-fbd8-079b-d78f49769a5b/data/data2/current/BP-773003372-172.17.0.3-1733534552230 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T01:23:16,624 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T01:23:16,629 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-07T01:23:16,630 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T01:23:16,630 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T01:23:16,630 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T01:23:16,630 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81befd3d-fc22-c9ba-afcb-aebe759d35a9/hadoop.log.dir/,STOPPED} 2024-12-07T01:23:16,638 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T01:23:16,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down