2024-11-18 23:35:15,132 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-18 23:35:15,157 main DEBUG Took 0.016941 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 23:35:15,158 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 23:35:15,158 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 23:35:15,160 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 23:35:15,163 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,175 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 23:35:15,192 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,194 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,195 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,196 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,197 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,198 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,199 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,199 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,200 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,202 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,203 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,203 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,204 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,205 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,205 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,206 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,206 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,207 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,207 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,208 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,208 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,209 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,210 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:35:15,210 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,211 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 23:35:15,213 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:35:15,215 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 23:35:15,217 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 23:35:15,218 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 23:35:15,219 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 23:35:15,220 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 23:35:15,232 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 23:35:15,235 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 23:35:15,237 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 23:35:15,238 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 23:35:15,238 main DEBUG createAppenders(={Console}) 2024-11-18 23:35:15,239 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-18 23:35:15,240 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-18 23:35:15,240 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-18 23:35:15,242 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 23:35:15,242 main DEBUG OutputStream closed 2024-11-18 23:35:15,243 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 23:35:15,243 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 23:35:15,243 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-18 23:35:15,366 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 23:35:15,370 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 23:35:15,371 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 23:35:15,372 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 23:35:15,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 23:35:15,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 23:35:15,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 23:35:15,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 23:35:15,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 23:35:15,377 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 23:35:15,377 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 23:35:15,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 23:35:15,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 23:35:15,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 23:35:15,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 23:35:15,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 23:35:15,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 23:35:15,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 23:35:15,386 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 23:35:15,387 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-18 23:35:15,388 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 23:35:15,389 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-18T23:35:15,744 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa 2024-11-18 23:35:15,750 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 23:35:15,751 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T23:35:15,763 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-18T23:35:15,839 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T23:35:15,925 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T23:35:15,926 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T23:35:15,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T23:35:15,964 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6, deleteOnExit=true 2024-11-18T23:35:15,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T23:35:15,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/test.cache.data in system properties and HBase conf 2024-11-18T23:35:15,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T23:35:15,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/hadoop.log.dir in system properties and HBase conf 2024-11-18T23:35:15,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T23:35:15,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T23:35:15,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T23:35:16,103 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T23:35:16,231 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:35:16,237 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T23:35:16,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T23:35:16,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T23:35:16,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T23:35:16,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T23:35:16,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T23:35:16,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T23:35:16,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T23:35:16,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T23:35:16,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/nfs.dump.dir in system properties and HBase conf 2024-11-18T23:35:16,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/java.io.tmpdir in system properties and HBase conf 2024-11-18T23:35:16,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T23:35:16,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T23:35:16,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T23:35:18,338 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T23:35:18,438 INFO [Time-limited test {}] log.Log(170): Logging initialized @4395ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T23:35:18,524 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T23:35:18,613 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T23:35:18,675 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T23:35:18,675 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T23:35:18,677 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T23:35:18,695 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T23:35:18,723 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b5fa2d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/hadoop.log.dir/,AVAILABLE} 2024-11-18T23:35:18,724 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d655053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T23:35:18,977 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7fe9bfd3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/java.io.tmpdir/jetty-localhost-44151-hadoop-hdfs-3_4_1-tests_jar-_-any-14381115139980429757/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T23:35:18,985 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16695a38{HTTP/1.1, (http/1.1)}{localhost:44151} 2024-11-18T23:35:18,986 INFO [Time-limited test {}] server.Server(415): Started @4943ms 2024-11-18T23:35:19,879 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T23:35:19,890 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T23:35:19,904 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T23:35:19,905 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T23:35:19,905 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T23:35:19,906 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@302ee775{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/hadoop.log.dir/,AVAILABLE} 2024-11-18T23:35:19,907 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c68adb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T23:35:20,054 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@516a39dc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/java.io.tmpdir/jetty-localhost-33173-hadoop-hdfs-3_4_1-tests_jar-_-any-14158857373343928026/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T23:35:20,056 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@428881d6{HTTP/1.1, (http/1.1)}{localhost:33173} 2024-11-18T23:35:20,057 INFO [Time-limited test {}] server.Server(415): Started @6014ms 2024-11-18T23:35:20,131 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T23:35:20,424 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T23:35:20,436 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T23:35:20,449 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T23:35:20,449 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T23:35:20,449 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T23:35:20,451 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2599967d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/hadoop.log.dir/,AVAILABLE} 2024-11-18T23:35:20,452 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4925ca3e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T23:35:20,600 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4cc5338b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/java.io.tmpdir/jetty-localhost-46367-hadoop-hdfs-3_4_1-tests_jar-_-any-1384416966139323102/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T23:35:20,601 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1364e1fb{HTTP/1.1, (http/1.1)}{localhost:46367} 2024-11-18T23:35:20,601 INFO [Time-limited test {}] server.Server(415): Started @6558ms 2024-11-18T23:35:20,606 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T23:35:20,730 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T23:35:20,739 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T23:35:20,769 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T23:35:20,769 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T23:35:20,769 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T23:35:20,770 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@626fc3bb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/hadoop.log.dir/,AVAILABLE} 2024-11-18T23:35:20,771 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@147d1a17{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T23:35:20,926 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ed6f3ed{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/java.io.tmpdir/jetty-localhost-45113-hadoop-hdfs-3_4_1-tests_jar-_-any-11405455944876718122/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T23:35:20,927 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f154eb3{HTTP/1.1, (http/1.1)}{localhost:45113} 2024-11-18T23:35:20,927 INFO [Time-limited test {}] server.Server(415): Started @6884ms 2024-11-18T23:35:20,930 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T23:35:22,141 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data3/current/BP-330010681-172.17.0.2-1731972917172/current, will proceed with Du for space computation calculation, 2024-11-18T23:35:22,144 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data4/current/BP-330010681-172.17.0.2-1731972917172/current, will proceed with Du for space computation calculation, 2024-11-18T23:35:22,153 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data2/current/BP-330010681-172.17.0.2-1731972917172/current, will proceed with Du for space computation calculation, 2024-11-18T23:35:22,158 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data1/current/BP-330010681-172.17.0.2-1731972917172/current, will proceed with Du for space computation calculation, 2024-11-18T23:35:22,249 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T23:35:22,249 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T23:35:22,315 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data5/current/BP-330010681-172.17.0.2-1731972917172/current, will proceed with Du for space computation calculation, 2024-11-18T23:35:22,321 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data6/current/BP-330010681-172.17.0.2-1731972917172/current, will proceed with Du for space computation calculation, 2024-11-18T23:35:22,334 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed41c382d118ccc5 with lease ID 0x4dfe1a55f5ff3e6c: Processing first storage report for DS-abd90c94-af46-4db3-8b6f-f6a6466c166d from datanode DatanodeRegistration(127.0.0.1:36973, datanodeUuid=1706733d-e884-4ee7-9e2f-89cc9dfd17a3, infoPort=42157, infoSecurePort=0, ipcPort=40073, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172) 2024-11-18T23:35:22,336 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed41c382d118ccc5 with lease ID 0x4dfe1a55f5ff3e6c: from storage DS-abd90c94-af46-4db3-8b6f-f6a6466c166d node DatanodeRegistration(127.0.0.1:36973, datanodeUuid=1706733d-e884-4ee7-9e2f-89cc9dfd17a3, infoPort=42157, infoSecurePort=0, ipcPort=40073, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-18T23:35:22,337 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x285e153b1567342e with lease ID 0x4dfe1a55f5ff3e6d: Processing first storage report for DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03 from datanode DatanodeRegistration(127.0.0.1:35345, datanodeUuid=e6f83191-0b6b-44b4-89ac-e00870a62215, infoPort=42361, infoSecurePort=0, ipcPort=33425, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172) 2024-11-18T23:35:22,337 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x285e153b1567342e with lease ID 0x4dfe1a55f5ff3e6d: from storage DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03 node DatanodeRegistration(127.0.0.1:35345, datanodeUuid=e6f83191-0b6b-44b4-89ac-e00870a62215, infoPort=42361, infoSecurePort=0, ipcPort=33425, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T23:35:22,337 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed41c382d118ccc5 with lease ID 0x4dfe1a55f5ff3e6c: Processing first storage report for DS-0c895694-4e9f-4d96-9587-526992bd2524 from datanode DatanodeRegistration(127.0.0.1:36973, datanodeUuid=1706733d-e884-4ee7-9e2f-89cc9dfd17a3, infoPort=42157, infoSecurePort=0, ipcPort=40073, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172) 2024-11-18T23:35:22,337 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed41c382d118ccc5 with lease ID 0x4dfe1a55f5ff3e6c: from storage DS-0c895694-4e9f-4d96-9587-526992bd2524 node DatanodeRegistration(127.0.0.1:36973, datanodeUuid=1706733d-e884-4ee7-9e2f-89cc9dfd17a3, infoPort=42157, infoSecurePort=0, ipcPort=40073, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T23:35:22,340 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x285e153b1567342e with lease ID 0x4dfe1a55f5ff3e6d: Processing first storage report for DS-7feae5a9-8747-41b4-b542-068d20226ca3 from datanode DatanodeRegistration(127.0.0.1:35345, datanodeUuid=e6f83191-0b6b-44b4-89ac-e00870a62215, infoPort=42361, infoSecurePort=0, ipcPort=33425, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172) 2024-11-18T23:35:22,341 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x285e153b1567342e with lease ID 0x4dfe1a55f5ff3e6d: from storage DS-7feae5a9-8747-41b4-b542-068d20226ca3 node DatanodeRegistration(127.0.0.1:35345, datanodeUuid=e6f83191-0b6b-44b4-89ac-e00870a62215, infoPort=42361, infoSecurePort=0, ipcPort=33425, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T23:35:22,396 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T23:35:22,404 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20a79549fc6e1dcf with lease ID 0x4dfe1a55f5ff3e6e: Processing first storage report for DS-bd9a80b9-d9ca-453c-916a-d97a723d440f from datanode DatanodeRegistration(127.0.0.1:39707, datanodeUuid=a3b93d3a-3bd9-42e0-ba1e-40787e0210fd, infoPort=46723, infoSecurePort=0, ipcPort=36933, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172) 2024-11-18T23:35:22,404 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20a79549fc6e1dcf with lease ID 0x4dfe1a55f5ff3e6e: from storage DS-bd9a80b9-d9ca-453c-916a-d97a723d440f node DatanodeRegistration(127.0.0.1:39707, datanodeUuid=a3b93d3a-3bd9-42e0-ba1e-40787e0210fd, infoPort=46723, infoSecurePort=0, ipcPort=36933, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T23:35:22,405 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20a79549fc6e1dcf with lease ID 0x4dfe1a55f5ff3e6e: Processing first storage report for DS-0961c200-ac34-42bc-b5a0-6d0ac74d0d4c from datanode DatanodeRegistration(127.0.0.1:39707, datanodeUuid=a3b93d3a-3bd9-42e0-ba1e-40787e0210fd, infoPort=46723, infoSecurePort=0, ipcPort=36933, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172) 2024-11-18T23:35:22,406 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20a79549fc6e1dcf with lease ID 0x4dfe1a55f5ff3e6e: from storage DS-0961c200-ac34-42bc-b5a0-6d0ac74d0d4c node DatanodeRegistration(127.0.0.1:39707, datanodeUuid=a3b93d3a-3bd9-42e0-ba1e-40787e0210fd, infoPort=46723, infoSecurePort=0, ipcPort=36933, storageInfo=lv=-57;cid=testClusterID;nsid=384896491;c=1731972917172), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T23:35:22,417 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa 2024-11-18T23:35:22,550 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/zookeeper_0, clientPort=59409, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T23:35:22,580 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59409 2024-11-18T23:35:22,625 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:22,628 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:22,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741825_1001 (size=7) 2024-11-18T23:35:22,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741825_1001 (size=7) 2024-11-18T23:35:22,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741825_1001 (size=7) 2024-11-18T23:35:23,360 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b with version=8 2024-11-18T23:35:23,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/hbase-staging 2024-11-18T23:35:23,664 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9d96596e55a9:0 server-side Connection retries=45 2024-11-18T23:35:23,673 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:23,673 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:23,678 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:35:23,678 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:23,678 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:35:23,849 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T23:35:23,921 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T23:35:23,937 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T23:35:23,942 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:35:23,975 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 72859 (auto-detected) 2024-11-18T23:35:23,976 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T23:35:24,005 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40599 2024-11-18T23:35:24,032 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40599 connecting to ZooKeeper ensemble=127.0.0.1:59409 2024-11-18T23:35:24,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405990x0, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:35:24,119 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40599-0x101504d78350000 connected 2024-11-18T23:35:24,351 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:24,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:24,391 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:35:24,401 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b, hbase.cluster.distributed=false 2024-11-18T23:35:24,444 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:35:24,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40599 2024-11-18T23:35:24,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40599 2024-11-18T23:35:24,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40599 2024-11-18T23:35:24,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40599 2024-11-18T23:35:24,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40599 2024-11-18T23:35:24,613 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9d96596e55a9:0 server-side Connection retries=45 2024-11-18T23:35:24,615 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:24,616 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:24,616 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:35:24,616 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:24,616 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:35:24,620 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T23:35:24,627 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:35:24,629 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46105 2024-11-18T23:35:24,631 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46105 connecting to ZooKeeper ensemble=127.0.0.1:59409 2024-11-18T23:35:24,633 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:24,635 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:24,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461050x0, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:35:24,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461050x0, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:35:24,872 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46105-0x101504d78350001 connected 2024-11-18T23:35:24,877 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T23:35:24,890 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T23:35:24,901 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T23:35:24,909 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:35:24,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46105 2024-11-18T23:35:24,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46105 2024-11-18T23:35:24,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46105 2024-11-18T23:35:24,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46105 2024-11-18T23:35:24,920 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46105 2024-11-18T23:35:24,944 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9d96596e55a9:0 server-side Connection retries=45 2024-11-18T23:35:24,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:24,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:24,945 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:35:24,945 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:24,945 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:35:24,945 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T23:35:24,945 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:35:24,946 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33655 2024-11-18T23:35:24,949 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33655 connecting to ZooKeeper ensemble=127.0.0.1:59409 2024-11-18T23:35:24,950 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:24,952 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:24,973 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:336550x0, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:35:24,974 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33655-0x101504d78350002 connected 2024-11-18T23:35:24,976 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:35:24,977 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T23:35:24,980 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T23:35:24,981 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T23:35:24,984 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:35:24,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33655 2024-11-18T23:35:24,989 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33655 2024-11-18T23:35:24,989 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33655 2024-11-18T23:35:24,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33655 2024-11-18T23:35:24,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33655 2024-11-18T23:35:25,009 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9d96596e55a9:0 server-side Connection retries=45 2024-11-18T23:35:25,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:25,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:25,009 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:35:25,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:35:25,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:35:25,010 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T23:35:25,010 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:35:25,011 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46043 2024-11-18T23:35:25,013 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46043 connecting to ZooKeeper ensemble=127.0.0.1:59409 2024-11-18T23:35:25,014 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:25,016 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:25,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460430x0, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:35:25,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46043-0x101504d78350003 connected 2024-11-18T23:35:25,056 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:35:25,057 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T23:35:25,058 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T23:35:25,059 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T23:35:25,061 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:35:25,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46043 2024-11-18T23:35:25,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46043 2024-11-18T23:35:25,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46043 2024-11-18T23:35:25,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46043 2024-11-18T23:35:25,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46043 2024-11-18T23:35:25,084 DEBUG [M:0;9d96596e55a9:40599 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9d96596e55a9:40599 2024-11-18T23:35:25,088 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9d96596e55a9,40599,1731972923486 2024-11-18T23:35:25,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:35:25,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:35:25,107 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:35:25,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:35:25,113 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9d96596e55a9,40599,1731972923486 2024-11-18T23:35:25,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:25,147 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T23:35:25,147 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:25,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T23:35:25,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T23:35:25,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:25,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:25,148 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T23:35:25,150 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9d96596e55a9,40599,1731972923486 from backup master directory 2024-11-18T23:35:25,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9d96596e55a9,40599,1731972923486 2024-11-18T23:35:25,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:35:25,163 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:35:25,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:35:25,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:35:25,164 WARN [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:35:25,165 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9d96596e55a9,40599,1731972923486 2024-11-18T23:35:25,167 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T23:35:25,169 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T23:35:25,228 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/hbase.id] with ID: 0eef78ff-7039-4105-a286-74bff0b7861a 2024-11-18T23:35:25,228 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/.tmp/hbase.id 2024-11-18T23:35:25,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741826_1002 (size=42) 2024-11-18T23:35:25,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741826_1002 (size=42) 2024-11-18T23:35:25,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741826_1002 (size=42) 2024-11-18T23:35:25,245 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/.tmp/hbase.id]:[hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/hbase.id] 2024-11-18T23:35:25,299 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:35:25,303 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T23:35:25,320 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-18T23:35:25,390 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:25,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:25,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:25,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:25,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741827_1003 (size=196) 2024-11-18T23:35:25,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741827_1003 (size=196) 2024-11-18T23:35:25,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741827_1003 (size=196) 2024-11-18T23:35:25,474 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:35:25,477 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T23:35:25,485 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:25,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741828_1004 (size=1189) 2024-11-18T23:35:25,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741828_1004 (size=1189) 2024-11-18T23:35:25,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741828_1004 (size=1189) 2024-11-18T23:35:25,981 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store 2024-11-18T23:35:26,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741829_1005 (size=34) 2024-11-18T23:35:26,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741829_1005 (size=34) 2024-11-18T23:35:26,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741829_1005 (size=34) 2024-11-18T23:35:26,026 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T23:35:26,030 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:26,031 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T23:35:26,031 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:35:26,032 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:35:26,033 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T23:35:26,033 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:35:26,033 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:35:26,035 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731972926031Disabling compacts and flushes for region at 1731972926031Disabling writes for close at 1731972926033 (+2 ms)Writing region close event to WAL at 1731972926033Closed at 1731972926033 2024-11-18T23:35:26,038 WARN [master/9d96596e55a9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/.initializing 2024-11-18T23:35:26,038 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/WALs/9d96596e55a9,40599,1731972923486 2024-11-18T23:35:26,048 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:26,069 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d96596e55a9%2C40599%2C1731972923486, suffix=, logDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/WALs/9d96596e55a9,40599,1731972923486, archiveDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/oldWALs, maxLogs=10 2024-11-18T23:35:26,109 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/WALs/9d96596e55a9,40599,1731972923486/9d96596e55a9%2C40599%2C1731972923486.1731972926076, exclude list is [], retry=0 2024-11-18T23:35:26,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:26,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:26,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:26,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T23:35:26,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:26,195 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/WALs/9d96596e55a9,40599,1731972923486/9d96596e55a9%2C40599%2C1731972923486.1731972926076 2024-11-18T23:35:26,200 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:35:26,201 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:26,201 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:26,209 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,210 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,273 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,311 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T23:35:26,315 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:26,318 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:35:26,319 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,328 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T23:35:26,328 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:26,330 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:26,330 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T23:35:26,335 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:26,337 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:26,337 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T23:35:26,347 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:26,348 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:26,349 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,354 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,356 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,364 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,364 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,370 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:35:26,378 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:35:26,384 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:26,389 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75246450, jitterRate=0.12125948071479797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:35:26,400 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731972926227Initializing all the Stores at 1731972926231 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731972926232 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972926235 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972926235Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972926236 (+1 ms)Cleaning up temporary data from old regions at 1731972926365 (+129 ms)Region opened successfully at 1731972926400 (+35 ms) 2024-11-18T23:35:26,409 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T23:35:26,486 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a2325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9d96596e55a9/172.17.0.2:0 2024-11-18T23:35:26,537 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T23:35:26,560 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T23:35:26,562 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T23:35:26,567 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T23:35:26,572 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 5 msec 2024-11-18T23:35:26,579 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-18T23:35:26,579 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T23:35:26,640 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T23:35:26,654 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T23:35:26,668 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T23:35:26,673 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T23:35:26,684 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T23:35:26,696 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T23:35:26,701 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T23:35:26,722 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T23:35:26,736 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T23:35:26,745 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T23:35:26,756 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T23:35:26,789 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T23:35:26,800 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T23:35:26,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:35:26,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:35:26,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:35:26,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,826 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:35:26,826 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,831 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9d96596e55a9,40599,1731972923486, sessionid=0x101504d78350000, setting cluster-up flag (Was=false) 2024-11-18T23:35:26,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,862 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,893 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T23:35:26,895 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9d96596e55a9,40599,1731972923486 2024-11-18T23:35:26,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,925 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:26,956 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T23:35:26,958 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9d96596e55a9,40599,1731972923486 2024-11-18T23:35:26,966 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T23:35:26,979 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(746): ClusterId : 0eef78ff-7039-4105-a286-74bff0b7861a 2024-11-18T23:35:26,980 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(746): ClusterId : 0eef78ff-7039-4105-a286-74bff0b7861a 2024-11-18T23:35:26,981 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(746): ClusterId : 0eef78ff-7039-4105-a286-74bff0b7861a 2024-11-18T23:35:26,982 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T23:35:26,982 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T23:35:26,982 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T23:35:27,011 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T23:35:27,011 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T23:35:27,011 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T23:35:27,011 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T23:35:27,012 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T23:35:27,012 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T23:35:27,034 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T23:35:27,035 DEBUG [RS:1;9d96596e55a9:33655 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519fdb09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9d96596e55a9/172.17.0.2:0 2024-11-18T23:35:27,041 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T23:35:27,041 DEBUG [RS:2;9d96596e55a9:46043 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@727a5479, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9d96596e55a9/172.17.0.2:0 2024-11-18T23:35:27,056 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T23:35:27,057 DEBUG [RS:0;9d96596e55a9:46105 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276202a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9d96596e55a9/172.17.0.2:0 2024-11-18T23:35:27,069 DEBUG [RS:1;9d96596e55a9:33655 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9d96596e55a9:33655 2024-11-18T23:35:27,069 DEBUG [RS:2;9d96596e55a9:46043 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9d96596e55a9:46043 2024-11-18T23:35:27,073 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T23:35:27,073 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T23:35:27,073 DEBUG [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T23:35:27,075 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T23:35:27,076 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T23:35:27,076 DEBUG [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T23:35:27,079 DEBUG [RS:0;9d96596e55a9:46105 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9d96596e55a9:46105 2024-11-18T23:35:27,079 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T23:35:27,080 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T23:35:27,080 DEBUG [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T23:35:27,087 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(2659): reportForDuty to master=9d96596e55a9,40599,1731972923486 with port=46105, startcode=1731972924571 2024-11-18T23:35:27,088 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(2659): reportForDuty to master=9d96596e55a9,40599,1731972923486 with port=46043, startcode=1731972925008 2024-11-18T23:35:27,088 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T23:35:27,095 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(2659): reportForDuty to master=9d96596e55a9,40599,1731972923486 with port=33655, startcode=1731972924943 2024-11-18T23:35:27,104 DEBUG [RS:1;9d96596e55a9:33655 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T23:35:27,104 DEBUG [RS:0;9d96596e55a9:46105 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T23:35:27,104 DEBUG [RS:2;9d96596e55a9:46043 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T23:35:27,106 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T23:35:27,123 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T23:35:27,134 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9d96596e55a9,40599,1731972923486 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T23:35:27,173 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37515, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T23:35:27,175 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47375, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T23:35:27,175 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45985, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T23:35:27,185 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9d96596e55a9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:35:27,185 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:35:27,186 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9d96596e55a9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:35:27,186 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9d96596e55a9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:35:27,186 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9d96596e55a9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:35:27,186 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9d96596e55a9:0, corePoolSize=10, maxPoolSize=10 2024-11-18T23:35:27,187 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,187 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9d96596e55a9:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:35:27,187 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,193 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:35:27,198 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:35:27,219 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731972957219 2024-11-18T23:35:27,222 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:35:27,223 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T23:35:27,223 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T23:35:27,228 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T23:35:27,235 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T23:35:27,235 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T23:35:27,236 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T23:35:27,236 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T23:35:27,238 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:27,237 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,239 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T23:35:27,245 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T23:35:27,245 DEBUG [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T23:35:27,245 WARN [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-18T23:35:27,246 DEBUG [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T23:35:27,246 WARN [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-18T23:35:27,246 DEBUG [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T23:35:27,246 WARN [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-18T23:35:27,246 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T23:35:27,248 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T23:35:27,257 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T23:35:27,260 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T23:35:27,264 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.large.0-1731972927262,5,FailOnTimeoutGroup] 2024-11-18T23:35:27,276 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.small.0-1731972927265,5,FailOnTimeoutGroup] 2024-11-18T23:35:27,276 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,278 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T23:35:27,281 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,282 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,347 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(2659): reportForDuty to master=9d96596e55a9,40599,1731972923486 with port=33655, startcode=1731972924943 2024-11-18T23:35:27,349 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9d96596e55a9,33655,1731972924943 2024-11-18T23:35:27,350 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(2659): reportForDuty to master=9d96596e55a9,40599,1731972923486 with port=46105, startcode=1731972924571 2024-11-18T23:35:27,353 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(2659): reportForDuty to master=9d96596e55a9,40599,1731972923486 with port=46043, startcode=1731972925008 2024-11-18T23:35:27,353 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(517): Registering regionserver=9d96596e55a9,33655,1731972924943 2024-11-18T23:35:27,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741831_1007 (size=1321) 2024-11-18T23:35:27,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741831_1007 (size=1321) 2024-11-18T23:35:27,369 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9d96596e55a9,46105,1731972924571 2024-11-18T23:35:27,369 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(517): Registering regionserver=9d96596e55a9,46105,1731972924571 2024-11-18T23:35:27,378 DEBUG [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b 2024-11-18T23:35:27,378 DEBUG [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37995 2024-11-18T23:35:27,378 DEBUG [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T23:35:27,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741831_1007 (size=1321) 2024-11-18T23:35:27,381 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9d96596e55a9,46043,1731972925008 2024-11-18T23:35:27,381 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(517): Registering regionserver=9d96596e55a9,46043,1731972925008 2024-11-18T23:35:27,385 DEBUG [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b 2024-11-18T23:35:27,386 DEBUG [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37995 2024-11-18T23:35:27,386 DEBUG [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T23:35:27,387 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T23:35:27,388 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b 2024-11-18T23:35:27,393 DEBUG [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b 2024-11-18T23:35:27,393 DEBUG [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37995 2024-11-18T23:35:27,393 DEBUG [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T23:35:27,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:35:27,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741832_1008 (size=32) 2024-11-18T23:35:27,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741832_1008 (size=32) 2024-11-18T23:35:27,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741832_1008 (size=32) 2024-11-18T23:35:27,462 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:27,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T23:35:27,476 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:35:27,476 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:27,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:35:27,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T23:35:27,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:35:27,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:27,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:35:27,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T23:35:27,496 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:35:27,496 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:27,500 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:35:27,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T23:35:27,505 DEBUG [RS:1;9d96596e55a9:33655 {}] zookeeper.ZKUtil(111): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9d96596e55a9,33655,1731972924943 2024-11-18T23:35:27,505 WARN [RS:1;9d96596e55a9:33655 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:35:27,505 INFO [RS:1;9d96596e55a9:33655 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:27,505 DEBUG [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,33655,1731972924943 2024-11-18T23:35:27,505 DEBUG [RS:2;9d96596e55a9:46043 {}] zookeeper.ZKUtil(111): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9d96596e55a9,46043,1731972925008 2024-11-18T23:35:27,505 WARN [RS:2;9d96596e55a9:46043 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:35:27,506 INFO [RS:2;9d96596e55a9:46043 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:27,506 DEBUG [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46043,1731972925008 2024-11-18T23:35:27,506 DEBUG [RS:0;9d96596e55a9:46105 {}] zookeeper.ZKUtil(111): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9d96596e55a9,46105,1731972924571 2024-11-18T23:35:27,507 WARN [RS:0;9d96596e55a9:46105 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:35:27,507 INFO [RS:0;9d96596e55a9:46105 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:27,507 DEBUG [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571 2024-11-18T23:35:27,508 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9d96596e55a9,46043,1731972925008] 2024-11-18T23:35:27,508 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9d96596e55a9,33655,1731972924943] 2024-11-18T23:35:27,508 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9d96596e55a9,46105,1731972924571] 2024-11-18T23:35:27,509 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:35:27,510 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:27,511 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:35:27,512 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:35:27,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740 2024-11-18T23:35:27,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740 2024-11-18T23:35:27,533 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:35:27,533 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:35:27,537 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:35:27,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:35:27,563 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:27,565 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61876114, jitterRate=-0.07797405123710632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:35:27,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731972927463Initializing all the Stores at 1731972927466 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731972927466Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731972927468 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972927468Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731972927468Cleaning up temporary data from old regions at 1731972927533 (+65 ms)Region opened successfully at 1731972927570 (+37 ms) 2024-11-18T23:35:27,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:35:27,571 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:35:27,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:35:27,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:35:27,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:35:27,575 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:35:27,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731972927571Disabling compacts and flushes for region at 1731972927571Disabling writes for close at 1731972927571Writing region close event to WAL at 1731972927575 (+4 ms)Closed at 1731972927575 2024-11-18T23:35:27,587 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T23:35:27,588 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T23:35:27,589 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:35:27,589 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T23:35:27,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T23:35:27,602 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T23:35:27,617 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:35:27,622 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T23:35:27,625 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T23:35:27,632 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T23:35:27,639 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T23:35:27,641 INFO [RS:2;9d96596e55a9:46043 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:35:27,641 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,641 INFO [RS:0;9d96596e55a9:46105 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:35:27,641 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,644 INFO [RS:1;9d96596e55a9:33655 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:35:27,645 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,649 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T23:35:27,650 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T23:35:27,652 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T23:35:27,659 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T23:35:27,660 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T23:35:27,662 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,662 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,662 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,662 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,663 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,663 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,663 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,663 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,663 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,663 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,663 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,663 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:35:27,663 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,663 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,663 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:35:27,663 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,664 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,664 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,664 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,664 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,664 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,664 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,665 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,665 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9d96596e55a9:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:35:27,665 DEBUG [RS:1;9d96596e55a9:33655 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:35:27,666 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T23:35:27,667 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,667 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,667 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,667 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,668 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,668 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,664 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,668 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:35:27,668 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,668 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,668 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,668 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,668 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,668 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9d96596e55a9:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:35:27,669 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,669 DEBUG [RS:0;9d96596e55a9:46105 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:35:27,669 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,669 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:35:27,669 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9d96596e55a9:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:35:27,669 DEBUG [RS:2;9d96596e55a9:46043 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:35:27,680 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,680 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,681 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,681 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,681 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,681 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,33655,1731972924943-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:35:27,722 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,722 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,722 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,722 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,722 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,723 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,46043,1731972925008-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:35:27,745 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T23:35:27,747 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,33655,1731972924943-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,747 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,748 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.Replication(171): 9d96596e55a9,33655,1731972924943 started 2024-11-18T23:35:27,748 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,748 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,748 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,748 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,748 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,749 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,46105,1731972924571-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:35:27,752 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T23:35:27,752 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,46043,1731972925008-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,769 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,769 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.Replication(171): 9d96596e55a9,46043,1731972925008 started 2024-11-18T23:35:27,785 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T23:35:27,785 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,46105,1731972924571-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,785 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,786 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.Replication(171): 9d96596e55a9,46105,1731972924571 started 2024-11-18T23:35:27,788 WARN [9d96596e55a9:40599 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T23:35:27,799 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,809 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(1482): Serving as 9d96596e55a9,33655,1731972924943, RpcServer on 9d96596e55a9/172.17.0.2:33655, sessionid=0x101504d78350002 2024-11-18T23:35:27,810 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T23:35:27,810 DEBUG [RS:1;9d96596e55a9:33655 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9d96596e55a9,33655,1731972924943 2024-11-18T23:35:27,810 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9d96596e55a9,33655,1731972924943' 2024-11-18T23:35:27,810 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T23:35:27,814 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T23:35:27,821 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T23:35:27,821 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T23:35:27,821 DEBUG [RS:1;9d96596e55a9:33655 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9d96596e55a9,33655,1731972924943 2024-11-18T23:35:27,821 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9d96596e55a9,33655,1731972924943' 2024-11-18T23:35:27,821 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T23:35:27,829 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T23:35:27,838 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,838 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(1482): Serving as 9d96596e55a9,46043,1731972925008, RpcServer on 9d96596e55a9/172.17.0.2:46043, sessionid=0x101504d78350003 2024-11-18T23:35:27,839 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T23:35:27,839 DEBUG [RS:2;9d96596e55a9:46043 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9d96596e55a9,46043,1731972925008 2024-11-18T23:35:27,839 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9d96596e55a9,46043,1731972925008' 2024-11-18T23:35:27,839 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T23:35:27,840 DEBUG [RS:1;9d96596e55a9:33655 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T23:35:27,840 INFO [RS:1;9d96596e55a9:33655 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T23:35:27,840 INFO [RS:1;9d96596e55a9:33655 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T23:35:27,851 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:27,851 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(1482): Serving as 9d96596e55a9,46105,1731972924571, RpcServer on 9d96596e55a9/172.17.0.2:46105, sessionid=0x101504d78350001 2024-11-18T23:35:27,851 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T23:35:27,851 DEBUG [RS:0;9d96596e55a9:46105 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9d96596e55a9,46105,1731972924571 2024-11-18T23:35:27,851 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9d96596e55a9,46105,1731972924571' 2024-11-18T23:35:27,852 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T23:35:27,852 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T23:35:27,859 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T23:35:27,859 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T23:35:27,860 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T23:35:27,860 DEBUG [RS:2;9d96596e55a9:46043 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9d96596e55a9,46043,1731972925008 2024-11-18T23:35:27,861 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9d96596e55a9,46043,1731972925008' 2024-11-18T23:35:27,861 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T23:35:27,869 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T23:35:27,869 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T23:35:27,870 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T23:35:27,872 DEBUG [RS:0;9d96596e55a9:46105 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9d96596e55a9,46105,1731972924571 2024-11-18T23:35:27,872 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9d96596e55a9,46105,1731972924571' 2024-11-18T23:35:27,873 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T23:35:27,877 DEBUG [RS:2;9d96596e55a9:46043 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T23:35:27,877 INFO [RS:2;9d96596e55a9:46043 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T23:35:27,877 INFO [RS:2;9d96596e55a9:46043 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T23:35:27,882 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T23:35:27,884 DEBUG [RS:0;9d96596e55a9:46105 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T23:35:27,884 INFO [RS:0;9d96596e55a9:46105 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T23:35:27,884 INFO [RS:0;9d96596e55a9:46105 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T23:35:27,947 INFO [RS:1;9d96596e55a9:33655 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:27,951 INFO [RS:1;9d96596e55a9:33655 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d96596e55a9%2C33655%2C1731972924943, suffix=, logDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,33655,1731972924943, archiveDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/oldWALs, maxLogs=32 2024-11-18T23:35:27,979 INFO [RS:2;9d96596e55a9:46043 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:27,980 DEBUG [RS:1;9d96596e55a9:33655 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,33655,1731972924943/9d96596e55a9%2C33655%2C1731972924943.1731972927957, exclude list is [], retry=0 2024-11-18T23:35:27,985 INFO [RS:0;9d96596e55a9:46105 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:27,990 INFO [RS:2;9d96596e55a9:46043 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d96596e55a9%2C46043%2C1731972925008, suffix=, logDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46043,1731972925008, archiveDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/oldWALs, maxLogs=32 2024-11-18T23:35:27,990 INFO [RS:0;9d96596e55a9:46105 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d96596e55a9%2C46105%2C1731972924571, suffix=, logDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571, archiveDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/oldWALs, maxLogs=32 2024-11-18T23:35:28,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:28,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:28,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:28,020 DEBUG [RS:0;9d96596e55a9:46105 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571/9d96596e55a9%2C46105%2C1731972924571.1731972927993, exclude list is [], retry=0 2024-11-18T23:35:28,020 DEBUG [RS:2;9d96596e55a9:46043 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46043,1731972925008/9d96596e55a9%2C46043%2C1731972925008.1731972927993, exclude list is [], retry=0 2024-11-18T23:35:28,022 INFO [RS:1;9d96596e55a9:33655 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,33655,1731972924943/9d96596e55a9%2C33655%2C1731972924943.1731972927957 2024-11-18T23:35:28,024 DEBUG [RS:1;9d96596e55a9:33655 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:42157:42157)] 2024-11-18T23:35:28,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:28,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:28,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:28,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:28,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:28,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:28,062 INFO [RS:2;9d96596e55a9:46043 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46043,1731972925008/9d96596e55a9%2C46043%2C1731972925008.1731972927993 2024-11-18T23:35:28,064 DEBUG [RS:2;9d96596e55a9:46043 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:35:28,077 INFO [RS:0;9d96596e55a9:46105 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571/9d96596e55a9%2C46105%2C1731972924571.1731972927993 2024-11-18T23:35:28,080 DEBUG [RS:0;9d96596e55a9:46105 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:42157:42157)] 2024-11-18T23:35:28,291 DEBUG [9d96596e55a9:40599 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-18T23:35:28,302 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(204): Hosts are {9d96596e55a9=0} racks are {/default-rack=0} 2024-11-18T23:35:28,311 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T23:35:28,311 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T23:35:28,311 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-18T23:35:28,311 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T23:35:28,311 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T23:35:28,312 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-18T23:35:28,312 INFO [9d96596e55a9:40599 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T23:35:28,312 INFO [9d96596e55a9:40599 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T23:35:28,312 INFO [9d96596e55a9:40599 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-18T23:35:28,312 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T23:35:28,322 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9d96596e55a9,33655,1731972924943 2024-11-18T23:35:28,331 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9d96596e55a9,33655,1731972924943, state=OPENING 2024-11-18T23:35:28,352 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T23:35:28,363 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:28,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:28,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:28,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:35:28,364 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:35:28,364 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:35:28,364 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:35:28,364 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:35:28,367 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:35:28,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9d96596e55a9,33655,1731972924943}] 2024-11-18T23:35:28,554 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:35:28,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43867, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:35:28,581 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T23:35:28,582 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:28,583 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T23:35:28,593 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d96596e55a9%2C33655%2C1731972924943.meta, suffix=.meta, logDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,33655,1731972924943, archiveDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/oldWALs, maxLogs=32 2024-11-18T23:35:28,619 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,33655,1731972924943/9d96596e55a9%2C33655%2C1731972924943.meta.1731972928597.meta, exclude list is [], retry=0 2024-11-18T23:35:28,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:28,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:28,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:28,646 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,33655,1731972924943/9d96596e55a9%2C33655%2C1731972924943.meta.1731972928597.meta 2024-11-18T23:35:28,648 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157)] 2024-11-18T23:35:28,649 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:28,652 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T23:35:28,655 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T23:35:28,661 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T23:35:28,666 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T23:35:28,667 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:28,667 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T23:35:28,667 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T23:35:28,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T23:35:28,686 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:35:28,686 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:28,689 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:35:28,689 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T23:35:28,691 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:35:28,692 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:28,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:35:28,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T23:35:28,697 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:35:28,697 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:28,698 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:35:28,698 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T23:35:28,701 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:35:28,701 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:28,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:35:28,702 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:35:28,704 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740 2024-11-18T23:35:28,708 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740 2024-11-18T23:35:28,712 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:35:28,712 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:35:28,713 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:35:28,717 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:35:28,720 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71987064, jitterRate=0.07269084453582764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:35:28,720 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T23:35:28,722 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731972928668Writing region info on filesystem at 1731972928668Initializing all the Stores at 1731972928677 (+9 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731972928677Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731972928680 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972928680Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731972928680Cleaning up temporary data from old regions at 1731972928712 (+32 ms)Running coprocessor post-open hooks at 1731972928720 (+8 ms)Region opened successfully at 1731972928722 (+2 ms) 2024-11-18T23:35:28,736 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731972928547 2024-11-18T23:35:28,758 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T23:35:28,759 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T23:35:28,767 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9d96596e55a9,33655,1731972924943 2024-11-18T23:35:28,771 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9d96596e55a9,33655,1731972924943, state=OPEN 2024-11-18T23:35:28,788 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:35:28,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:35:28,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:35:28,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:35:28,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:35:28,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:35:28,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:35:28,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:35:28,793 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9d96596e55a9,33655,1731972924943 2024-11-18T23:35:28,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T23:35:28,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9d96596e55a9,33655,1731972924943 in 423 msec 2024-11-18T23:35:28,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T23:35:28,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.2360 sec 2024-11-18T23:35:28,851 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:35:28,851 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T23:35:28,875 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:35:28,883 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,33655,1731972924943, seqNum=-1] 2024-11-18T23:35:28,958 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:35:28,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37019, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:35:29,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 2.0250 sec 2024-11-18T23:35:29,032 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731972929031, completionTime=-1 2024-11-18T23:35:29,035 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-18T23:35:29,036 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-18T23:35:29,069 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-18T23:35:29,069 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731972989069 2024-11-18T23:35:29,069 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731973049069 2024-11-18T23:35:29,069 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 33 msec 2024-11-18T23:35:29,071 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-18T23:35:29,077 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,40599,1731972923486-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:29,081 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,40599,1731972923486-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:29,082 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,40599,1731972923486-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:29,083 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9d96596e55a9:40599, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:29,092 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:29,098 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:29,099 DEBUG [master/9d96596e55a9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T23:35:29,140 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.975sec 2024-11-18T23:35:29,143 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T23:35:29,145 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T23:35:29,146 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T23:35:29,146 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T23:35:29,147 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T23:35:29,147 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,40599,1731972923486-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:35:29,148 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,40599,1731972923486-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T23:35:29,170 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T23:35:29,171 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T23:35:29,172 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,40599,1731972923486-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:29,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b257743, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:35:29,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,40599,-1 for getting cluster id 2024-11-18T23:35:29,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:35:29,229 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0eef78ff-7039-4105-a286-74bff0b7861a' 2024-11-18T23:35:29,233 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:35:29,234 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0eef78ff-7039-4105-a286-74bff0b7861a" 2024-11-18T23:35:29,235 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ed12c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:35:29,235 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,40599,-1] 2024-11-18T23:35:29,238 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:35:29,242 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:35:29,247 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:35:29,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60e8ad49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:35:29,254 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:35:29,267 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,33655,1731972924943, seqNum=-1] 2024-11-18T23:35:29,268 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:35:29,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:35:29,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9d96596e55a9,40599,1731972923486 2024-11-18T23:35:29,308 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:37995/hbase 2024-11-18T23:35:29,343 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=610, MaxFileDescriptor=1048576, SystemLoadAverage=1628, ProcessCount=11, AvailableMemoryMB=5718 2024-11-18T23:35:29,381 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:29,391 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:29,394 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:29,400 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-25043099, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/hregion-25043099, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:29,421 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-25043099/hregion-25043099.1731972929402, exclude list is [], retry=0 2024-11-18T23:35:29,426 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:29,429 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:29,436 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:29,445 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-25043099/hregion-25043099.1731972929402 2024-11-18T23:35:29,448 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:35:29,448 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 62de49c721ca8a9d3265023b79f93158, NAME => 'testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/hbase 2024-11-18T23:35:29,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741838_1014 (size=64) 2024-11-18T23:35:29,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741838_1014 (size=64) 2024-11-18T23:35:29,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741838_1014 (size=64) 2024-11-18T23:35:29,482 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:29,485 INFO [StoreOpener-62de49c721ca8a9d3265023b79f93158-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:29,488 INFO [StoreOpener-62de49c721ca8a9d3265023b79f93158-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62de49c721ca8a9d3265023b79f93158 columnFamilyName a 2024-11-18T23:35:29,489 DEBUG [StoreOpener-62de49c721ca8a9d3265023b79f93158-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:29,490 INFO [StoreOpener-62de49c721ca8a9d3265023b79f93158-1 {}] regionserver.HStore(327): Store=62de49c721ca8a9d3265023b79f93158/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:29,490 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:29,492 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:29,493 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:29,494 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:29,494 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:29,498 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:29,502 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:29,503 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62de49c721ca8a9d3265023b79f93158; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64464697, jitterRate=-0.039401158690452576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:35:29,504 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62de49c721ca8a9d3265023b79f93158: Writing region info on filesystem at 1731972929482Initializing all the Stores at 1731972929484 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972929484Cleaning up temporary data from old regions at 1731972929494 (+10 ms)Region opened successfully at 1731972929504 (+10 ms) 2024-11-18T23:35:29,504 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 62de49c721ca8a9d3265023b79f93158, disabling compactions & flushes 2024-11-18T23:35:29,505 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158. 2024-11-18T23:35:29,505 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158. 2024-11-18T23:35:29,505 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158. after waiting 0 ms 2024-11-18T23:35:29,505 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158. 2024-11-18T23:35:29,512 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158. 2024-11-18T23:35:29,513 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 62de49c721ca8a9d3265023b79f93158: Waiting for close lock at 1731972929504Disabling compacts and flushes for region at 1731972929504Disabling writes for close at 1731972929505 (+1 ms)Writing region close event to WAL at 1731972929512 (+7 ms)Closed at 1731972929512 2024-11-18T23:35:29,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741837_1013 (size=93) 2024-11-18T23:35:29,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741837_1013 (size=93) 2024-11-18T23:35:29,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741837_1013 (size=93) 2024-11-18T23:35:29,547 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:35:29,548 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-25043099:(num 1731972929402) 2024-11-18T23:35:29,551 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-18T23:35:29,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741839_1015 (size=369) 2024-11-18T23:35:29,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741839_1015 (size=369) 2024-11-18T23:35:29,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741839_1015 (size=369) 2024-11-18T23:35:29,599 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-18T23:35:29,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741840_1016 (size=231) 2024-11-18T23:35:29,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741840_1016 (size=231) 2024-11-18T23:35:29,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741840_1016 (size=231) 2024-11-18T23:35:29,656 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-1, size=369 (369bytes) 2024-11-18T23:35:29,657 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-18T23:35:29,657 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-18T23:35:29,658 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-1 2024-11-18T23:35:29,677 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-1 after 12ms 2024-11-18T23:35:29,700 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-1 took 45ms 2024-11-18T23:35:29,713 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-1 so closing down 2024-11-18T23:35:29,718 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:29,728 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-18T23:35:29,731 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000001-wal-1.temp 2024-11-18T23:35:29,736 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:29,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741841_1017 (size=369) 2024-11-18T23:35:29,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741841_1017 (size=369) 2024-11-18T23:35:29,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741841_1017 (size=369) 2024-11-18T23:35:29,798 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-18T23:35:29,807 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000002 2024-11-18T23:35:29,818 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 100 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-18T23:35:29,818 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-1, journal: Splitting hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-1, size=369 (369bytes) at 1731972929656Finishing writing output for hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-1 so closing down at 1731972929713 (+57 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000001-wal-1.temp at 1731972929731 (+18 ms)3 split writer threads finished at 1731972929736 (+5 ms)Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731972929798 (+62 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000002 at 1731972929807 (+9 ms)Processed 2 edits across 1 Regions in 100 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731972929818 (+11 ms) 2024-11-18T23:35:29,847 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-2, size=231 (231bytes) 2024-11-18T23:35:29,847 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-2 2024-11-18T23:35:29,852 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-2 after 5ms 2024-11-18T23:35:29,867 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-2 took 20ms 2024-11-18T23:35:29,882 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-2 so closing down 2024-11-18T23:35:29,882 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:29,888 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-18T23:35:29,900 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000002-wal-2.temp 2024-11-18T23:35:29,901 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:29,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741842_1018 (size=231) 2024-11-18T23:35:29,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741842_1018 (size=231) 2024-11-18T23:35:29,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741842_1018 (size=231) 2024-11-18T23:35:30,389 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-18T23:35:30,398 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-18T23:35:30,400 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 524 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-18T23:35:30,400 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-2, journal: Splitting hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-2, size=231 (231bytes) at 1731972929847Finishing writing output for hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-2 so closing down at 1731972929882 (+35 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000002-wal-2.temp at 1731972929900 (+18 ms)3 split writer threads finished at 1731972929901 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731972930389 (+488 ms)Processed 1 edits across 1 Regions in 524 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731972930400 (+11 ms) 2024-11-18T23:35:30,400 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:30,403 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:30,423 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal.1731972930404, exclude list is [], retry=0 2024-11-18T23:35:30,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:30,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:30,429 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:30,441 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal.1731972930404 2024-11-18T23:35:30,445 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:35:30,445 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 62de49c721ca8a9d3265023b79f93158, NAME => 'testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:30,446 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:30,446 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:30,446 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:30,453 INFO [StoreOpener-62de49c721ca8a9d3265023b79f93158-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:30,468 INFO [StoreOpener-62de49c721ca8a9d3265023b79f93158-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62de49c721ca8a9d3265023b79f93158 columnFamilyName a 2024-11-18T23:35:30,468 DEBUG [StoreOpener-62de49c721ca8a9d3265023b79f93158-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:30,473 INFO [StoreOpener-62de49c721ca8a9d3265023b79f93158-1 {}] regionserver.HStore(327): Store=62de49c721ca8a9d3265023b79f93158/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:30,475 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:30,479 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:30,484 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:30,486 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000002 2024-11-18T23:35:30,496 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000002 2024-11-18T23:35:30,500 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 62de49c721ca8a9d3265023b79f93158 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-18T23:35:30,575 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/.tmp/a/071d3c57d1ad486987519fab69cae645 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731972929548/Put/seqid=0 2024-11-18T23:35:30,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741844_1020 (size=5170) 2024-11-18T23:35:30,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741844_1020 (size=5170) 2024-11-18T23:35:30,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741844_1020 (size=5170) 2024-11-18T23:35:30,589 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/.tmp/a/071d3c57d1ad486987519fab69cae645 2024-11-18T23:35:30,640 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/.tmp/a/071d3c57d1ad486987519fab69cae645 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/a/071d3c57d1ad486987519fab69cae645 2024-11-18T23:35:30,654 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/a/071d3c57d1ad486987519fab69cae645, entries=2, sequenceid=2, filesize=5.0 K 2024-11-18T23:35:30,660 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 62de49c721ca8a9d3265023b79f93158 in 160ms, sequenceid=2, compaction requested=false; wal=null 2024-11-18T23:35:30,662 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/0000000000000000002 2024-11-18T23:35:30,663 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:30,663 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:30,666 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62de49c721ca8a9d3265023b79f93158 2024-11-18T23:35:30,671 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/62de49c721ca8a9d3265023b79f93158/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-18T23:35:30,673 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62de49c721ca8a9d3265023b79f93158; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68635657, jitterRate=0.022750988602638245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:35:30,674 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62de49c721ca8a9d3265023b79f93158: Writing region info on filesystem at 1731972930446Initializing all the Stores at 1731972930448 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972930448Obtaining lock to block concurrent updates at 1731972930500 (+52 ms)Preparing flush snapshotting stores in 62de49c721ca8a9d3265023b79f93158 at 1731972930500Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731972930506 (+6 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731972929386.62de49c721ca8a9d3265023b79f93158. at 1731972930506Flushing 62de49c721ca8a9d3265023b79f93158/a: creating writer at 1731972930508 (+2 ms)Flushing 62de49c721ca8a9d3265023b79f93158/a: appending metadata at 1731972930562 (+54 ms)Flushing 62de49c721ca8a9d3265023b79f93158/a: closing flushed file at 1731972930565 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38178104: reopening flushed file at 1731972930638 (+73 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 62de49c721ca8a9d3265023b79f93158 in 160ms, sequenceid=2, compaction requested=false; wal=null at 1731972930660 (+22 ms)Cleaning up temporary data from old regions at 1731972930663 (+3 ms)Region opened successfully at 1731972930674 (+11 ms) 2024-11-18T23:35:30,706 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=376 (was 365) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:54908 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:43864 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:44282 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:37995/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:44310 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:43778 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=700 (was 610) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1628 (was 1628), ProcessCount=11 (was 11), AvailableMemoryMB=5488 (was 5718) 2024-11-18T23:35:30,717 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=700, MaxFileDescriptor=1048576, SystemLoadAverage=1628, ProcessCount=11, AvailableMemoryMB=5486 2024-11-18T23:35:30,732 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:30,735 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:30,735 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:30,739 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-43522019, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/hregion-43522019, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:30,759 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-43522019/hregion-43522019.1731972930740, exclude list is [], retry=0 2024-11-18T23:35:30,764 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:30,765 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:30,769 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:30,779 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-43522019/hregion-43522019.1731972930740 2024-11-18T23:35:30,780 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:42157:42157)] 2024-11-18T23:35:30,780 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3e55184c31a57c18f09ba21ceb2c7ab2, NAME => 'testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/hbase 2024-11-18T23:35:30,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741846_1022 (size=64) 2024-11-18T23:35:30,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741846_1022 (size=64) 2024-11-18T23:35:30,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741846_1022 (size=64) 2024-11-18T23:35:30,793 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:30,795 INFO [StoreOpener-3e55184c31a57c18f09ba21ceb2c7ab2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:30,801 INFO [StoreOpener-3e55184c31a57c18f09ba21ceb2c7ab2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e55184c31a57c18f09ba21ceb2c7ab2 columnFamilyName a 2024-11-18T23:35:30,801 DEBUG [StoreOpener-3e55184c31a57c18f09ba21ceb2c7ab2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:30,802 INFO [StoreOpener-3e55184c31a57c18f09ba21ceb2c7ab2-1 {}] regionserver.HStore(327): Store=3e55184c31a57c18f09ba21ceb2c7ab2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:30,803 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:30,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:30,805 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:30,805 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:30,805 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:30,808 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:30,811 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:30,812 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3e55184c31a57c18f09ba21ceb2c7ab2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71537216, jitterRate=0.06598758697509766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:35:30,812 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3e55184c31a57c18f09ba21ceb2c7ab2: Writing region info on filesystem at 1731972930794Initializing all the Stores at 1731972930795 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972930795Cleaning up temporary data from old regions at 1731972930805 (+10 ms)Region opened successfully at 1731972930812 (+7 ms) 2024-11-18T23:35:30,812 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3e55184c31a57c18f09ba21ceb2c7ab2, disabling compactions & flushes 2024-11-18T23:35:30,812 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2. 2024-11-18T23:35:30,813 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2. 2024-11-18T23:35:30,813 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2. after waiting 0 ms 2024-11-18T23:35:30,813 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2. 2024-11-18T23:35:30,813 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2. 2024-11-18T23:35:30,813 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3e55184c31a57c18f09ba21ceb2c7ab2: Waiting for close lock at 1731972930812Disabling compacts and flushes for region at 1731972930812Disabling writes for close at 1731972930813 (+1 ms)Writing region close event to WAL at 1731972930813Closed at 1731972930813 2024-11-18T23:35:30,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741845_1021 (size=93) 2024-11-18T23:35:30,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741845_1021 (size=93) 2024-11-18T23:35:30,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741845_1021 (size=93) 2024-11-18T23:35:30,820 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:35:30,820 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-43522019:(num 1731972930740) 2024-11-18T23:35:30,821 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-18T23:35:30,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741847_1023 (size=369) 2024-11-18T23:35:30,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741847_1023 (size=369) 2024-11-18T23:35:30,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741847_1023 (size=369) 2024-11-18T23:35:30,834 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-18T23:35:30,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741848_1024 (size=231) 2024-11-18T23:35:30,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741848_1024 (size=231) 2024-11-18T23:35:30,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741848_1024 (size=231) 2024-11-18T23:35:30,882 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-2, size=231 (231bytes) 2024-11-18T23:35:30,882 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-2 2024-11-18T23:35:30,883 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-2 after 1ms 2024-11-18T23:35:30,887 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-2 took 6ms 2024-11-18T23:35:30,889 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-2 so closing down 2024-11-18T23:35:30,890 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:30,892 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-18T23:35:30,894 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002-wal-2.temp 2024-11-18T23:35:30,894 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:30,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741849_1025 (size=231) 2024-11-18T23:35:30,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741849_1025 (size=231) 2024-11-18T23:35:30,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741849_1025 (size=231) 2024-11-18T23:35:31,316 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-18T23:35:31,319 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002 2024-11-18T23:35:31,319 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 432 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-18T23:35:31,319 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-2, journal: Splitting hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-2, size=231 (231bytes) at 1731972930882Finishing writing output for hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-2 so closing down at 1731972930890 (+8 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002-wal-2.temp at 1731972930894 (+4 ms)3 split writer threads finished at 1731972930894Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731972931316 (+422 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002 at 1731972931319 (+3 ms)Processed 1 edits across 1 Regions in 432 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731972931319 2024-11-18T23:35:31,336 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-1, size=369 (369bytes) 2024-11-18T23:35:31,336 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-1 2024-11-18T23:35:31,337 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-1 after 1ms 2024-11-18T23:35:31,342 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-1 took 6ms 2024-11-18T23:35:31,346 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-1 so closing down 2024-11-18T23:35:31,346 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:31,348 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-18T23:35:31,349 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000001-wal-1.temp 2024-11-18T23:35:31,350 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:31,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741850_1026 (size=369) 2024-11-18T23:35:31,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741850_1026 (size=369) 2024-11-18T23:35:31,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741850_1026 (size=369) 2024-11-18T23:35:31,367 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-18T23:35:31,374 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002, length=231 2024-11-18T23:35:31,377 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002 2024-11-18T23:35:31,377 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-18T23:35:31,378 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-1, journal: Splitting hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-1, size=369 (369bytes) at 1731972931336Finishing writing output for hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-1 so closing down at 1731972931346 (+10 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000001-wal-1.temp at 1731972931349 (+3 ms)3 split writer threads finished at 1731972931350 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731972931367 (+17 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002 at 1731972931377 (+10 ms)Processed 2 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731972931377 2024-11-18T23:35:31,378 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:31,380 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:31,399 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal.1731972931381, exclude list is [], retry=0 2024-11-18T23:35:31,410 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:31,410 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:31,411 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:31,418 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal.1731972931381 2024-11-18T23:35:31,419 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:35:31,419 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3e55184c31a57c18f09ba21ceb2c7ab2, NAME => 'testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:31,419 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:31,419 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:31,419 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:31,425 INFO [StoreOpener-3e55184c31a57c18f09ba21ceb2c7ab2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:31,428 INFO [StoreOpener-3e55184c31a57c18f09ba21ceb2c7ab2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e55184c31a57c18f09ba21ceb2c7ab2 columnFamilyName a 2024-11-18T23:35:31,428 DEBUG [StoreOpener-3e55184c31a57c18f09ba21ceb2c7ab2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:31,429 INFO [StoreOpener-3e55184c31a57c18f09ba21ceb2c7ab2-1 {}] regionserver.HStore(327): Store=3e55184c31a57c18f09ba21ceb2c7ab2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:31,430 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:31,431 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:31,434 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:31,436 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002 2024-11-18T23:35:31,442 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002 2024-11-18T23:35:31,442 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3e55184c31a57c18f09ba21ceb2c7ab2 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-18T23:35:31,463 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/.tmp/a/78376833f39a405eb71de75d41c18fbe is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731972930820/Put/seqid=0 2024-11-18T23:35:31,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741852_1028 (size=5170) 2024-11-18T23:35:31,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741852_1028 (size=5170) 2024-11-18T23:35:31,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741852_1028 (size=5170) 2024-11-18T23:35:31,540 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/.tmp/a/78376833f39a405eb71de75d41c18fbe 2024-11-18T23:35:31,553 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/.tmp/a/78376833f39a405eb71de75d41c18fbe as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/a/78376833f39a405eb71de75d41c18fbe 2024-11-18T23:35:31,563 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/a/78376833f39a405eb71de75d41c18fbe, entries=2, sequenceid=2, filesize=5.0 K 2024-11-18T23:35:31,563 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3e55184c31a57c18f09ba21ceb2c7ab2 in 121ms, sequenceid=2, compaction requested=false; wal=null 2024-11-18T23:35:31,564 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/0000000000000000002 2024-11-18T23:35:31,565 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:31,565 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:31,569 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3e55184c31a57c18f09ba21ceb2c7ab2 2024-11-18T23:35:31,573 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/3e55184c31a57c18f09ba21ceb2c7ab2/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-18T23:35:31,575 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3e55184c31a57c18f09ba21ceb2c7ab2; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74866590, jitterRate=0.115599125623703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:35:31,575 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3e55184c31a57c18f09ba21ceb2c7ab2: Writing region info on filesystem at 1731972931419Initializing all the Stores at 1731972931421 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972931421Obtaining lock to block concurrent updates at 1731972931442 (+21 ms)Preparing flush snapshotting stores in 3e55184c31a57c18f09ba21ceb2c7ab2 at 1731972931442Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731972931442Flushing stores of testReplayEditsWrittenIntoWAL,,1731972930733.3e55184c31a57c18f09ba21ceb2c7ab2. at 1731972931442Flushing 3e55184c31a57c18f09ba21ceb2c7ab2/a: creating writer at 1731972931442Flushing 3e55184c31a57c18f09ba21ceb2c7ab2/a: appending metadata at 1731972931461 (+19 ms)Flushing 3e55184c31a57c18f09ba21ceb2c7ab2/a: closing flushed file at 1731972931461Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e9c0bdc: reopening flushed file at 1731972931551 (+90 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3e55184c31a57c18f09ba21ceb2c7ab2 in 121ms, sequenceid=2, compaction requested=false; wal=null at 1731972931563 (+12 ms)Cleaning up temporary data from old regions at 1731972931565 (+2 ms)Region opened successfully at 1731972931575 (+10 ms) 2024-11-18T23:35:31,597 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=387 (was 376) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:55002 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:43936 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:44378 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:43778 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:54976 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:44282 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 700) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1545 (was 1628), ProcessCount=11 (was 11), AvailableMemoryMB=5380 (was 5486) 2024-11-18T23:35:31,611 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=387, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=1545, ProcessCount=11, AvailableMemoryMB=5378 2024-11-18T23:35:31,632 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:31,635 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:31,636 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:31,643 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-83859580, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/hregion-83859580, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:31,662 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-83859580/hregion-83859580.1731972931643, exclude list is [], retry=0 2024-11-18T23:35:31,667 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:31,667 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:31,669 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:31,674 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-83859580/hregion-83859580.1731972931643 2024-11-18T23:35:31,674 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:35:31,675 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 83a955c880a10d3118e39eb8be5b22ee, NAME => 'testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/hbase 2024-11-18T23:35:31,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741854_1030 (size=64) 2024-11-18T23:35:31,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741854_1030 (size=64) 2024-11-18T23:35:31,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741854_1030 (size=64) 2024-11-18T23:35:31,710 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:31,713 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:31,716 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83a955c880a10d3118e39eb8be5b22ee columnFamilyName a 2024-11-18T23:35:31,717 DEBUG [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:31,718 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(327): Store=83a955c880a10d3118e39eb8be5b22ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:31,718 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:31,720 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83a955c880a10d3118e39eb8be5b22ee columnFamilyName b 2024-11-18T23:35:31,720 DEBUG [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:31,728 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(327): Store=83a955c880a10d3118e39eb8be5b22ee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:31,728 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:31,731 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83a955c880a10d3118e39eb8be5b22ee columnFamilyName c 2024-11-18T23:35:31,731 DEBUG [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:31,732 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(327): Store=83a955c880a10d3118e39eb8be5b22ee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:31,732 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:31,738 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:31,738 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:31,740 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:31,740 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:31,741 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:31,743 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:31,749 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:31,750 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 83a955c880a10d3118e39eb8be5b22ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66853928, jitterRate=-0.003798842430114746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:31,751 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 83a955c880a10d3118e39eb8be5b22ee: Writing region info on filesystem at 1731972931710Initializing all the Stores at 1731972931712 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972931712Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972931713 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972931713Cleaning up temporary data from old regions at 1731972931740 (+27 ms)Region opened successfully at 1731972931750 (+10 ms) 2024-11-18T23:35:31,751 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 83a955c880a10d3118e39eb8be5b22ee, disabling compactions & flushes 2024-11-18T23:35:31,751 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. 2024-11-18T23:35:31,751 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. 2024-11-18T23:35:31,751 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. after waiting 0 ms 2024-11-18T23:35:31,751 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. 2024-11-18T23:35:31,752 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. 2024-11-18T23:35:31,752 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 83a955c880a10d3118e39eb8be5b22ee: Waiting for close lock at 1731972931751Disabling compacts and flushes for region at 1731972931751Disabling writes for close at 1731972931751Writing region close event to WAL at 1731972931752 (+1 ms)Closed at 1731972931752 2024-11-18T23:35:31,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741853_1029 (size=93) 2024-11-18T23:35:31,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741853_1029 (size=93) 2024-11-18T23:35:31,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741853_1029 (size=93) 2024-11-18T23:35:31,768 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:35:31,768 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-83859580:(num 1731972931643) 2024-11-18T23:35:31,769 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:31,771 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:31,790 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772, exclude list is [], retry=0 2024-11-18T23:35:31,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:31,796 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:31,796 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:31,809 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 2024-11-18T23:35:31,811 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157)] 2024-11-18T23:35:32,133 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772, size=0 (0bytes) 2024-11-18T23:35:32,133 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 might be still open, length is 0 2024-11-18T23:35:32,133 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 2024-11-18T23:35:32,136 WARN [IPC Server handler 3 on default port 37995 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-18T23:35:32,138 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 after 4ms 2024-11-18T23:35:33,591 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T23:35:34,182 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:35:34,264 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T23:35:34,265 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T23:35:34,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T23:35:34,266 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T23:35:34,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T23:35:34,267 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T23:35:34,268 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-18T23:35:34,268 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-18T23:35:34,423 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:55048 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35345:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55048 dst: /127.0.0.1:35345 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35345 remote=/127.0.0.1:55048]. Total timeout mills is 60000, 57652 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:34,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:44422 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39707:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44422 dst: /127.0.0.1:39707 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:34,425 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:43992 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43992 dst: /127.0.0.1:36973 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:34,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741855_1032 (size=470675) 2024-11-18T23:35:34,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741855_1032 (size=470675) 2024-11-18T23:35:34,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741855_1032 (size=470675) 2024-11-18T23:35:36,138 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 after 4005ms 2024-11-18T23:35:36,144 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 took 4011ms 2024-11-18T23:35:36,153 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731972931772.temp 2024-11-18T23:35:36,155 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000000001-wal.1731972931772.temp 2024-11-18T23:35:36,231 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772; continuing. 2024-11-18T23:35:36,231 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 so closing down 2024-11-18T23:35:36,231 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:36,235 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:36,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741856_1033 (size=470683) 2024-11-18T23:35:36,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741856_1033 (size=470683) 2024-11-18T23:35:36,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741856_1033 (size=470683) 2024-11-18T23:35:36,643 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000000001-wal.1731972931772.temp (wrote 3002 edits, skipped 0 edits in 31 ms) 2024-11-18T23:35:36,645 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000000001-wal.1731972931772.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000003002 2024-11-18T23:35:36,645 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 501 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772, size=0, length=0, corrupted=false, cancelled=false 2024-11-18T23:35:36,645 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772, journal: Splitting hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772, size=0 (0bytes) at 1731972932133Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000000001-wal.1731972931772.temp at 1731972936155 (+4022 ms)Split 1024 edits, skipped 0 edits. at 1731972936203 (+48 ms)Split 2048 edits, skipped 0 edits. at 1731972936220 (+17 ms)Finishing writing output for hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 so closing down at 1731972936231 (+11 ms)3 split writer threads finished at 1731972936235 (+4 ms)Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000000001-wal.1731972931772.temp (wrote 3002 edits, skipped 0 edits in 31 ms) at 1731972936643 (+408 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000000001-wal.1731972931772.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000003002 at 1731972936645 (+2 ms)Processed 3002 edits across 1 Regions in 501 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772, size=0, length=0, corrupted=false, cancelled=false at 1731972936645 2024-11-18T23:35:36,648 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 to hdfs://localhost:37995/hbase/oldWALs/wal.1731972931772 2024-11-18T23:35:36,649 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000003002 2024-11-18T23:35:36,649 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:36,652 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:36,668 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972936652, exclude list is [], retry=0 2024-11-18T23:35:36,672 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:36,673 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:36,673 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:36,696 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972936652 2024-11-18T23:35:36,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:35:36,698 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:36,700 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:36,705 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83a955c880a10d3118e39eb8be5b22ee columnFamilyName a 2024-11-18T23:35:36,705 DEBUG [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:36,708 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(327): Store=83a955c880a10d3118e39eb8be5b22ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:36,708 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:36,710 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83a955c880a10d3118e39eb8be5b22ee columnFamilyName b 2024-11-18T23:35:36,710 DEBUG [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:36,710 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(327): Store=83a955c880a10d3118e39eb8be5b22ee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:36,711 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:36,712 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83a955c880a10d3118e39eb8be5b22ee columnFamilyName c 2024-11-18T23:35:36,712 DEBUG [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:36,714 INFO [StoreOpener-83a955c880a10d3118e39eb8be5b22ee-1 {}] regionserver.HStore(327): Store=83a955c880a10d3118e39eb8be5b22ee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:36,714 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:36,715 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:36,722 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:36,723 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000003002 2024-11-18T23:35:36,785 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-18T23:35:37,308 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 83a955c880a10d3118e39eb8be5b22ee 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-18T23:35:37,375 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/a/377a9d630c754c5f8979c595b1f0a2e0 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731972931821/Put/seqid=0 2024-11-18T23:35:37,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741858_1035 (size=50463) 2024-11-18T23:35:37,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741858_1035 (size=50463) 2024-11-18T23:35:37,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741858_1035 (size=50463) 2024-11-18T23:35:37,404 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/a/377a9d630c754c5f8979c595b1f0a2e0 2024-11-18T23:35:37,418 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/a/377a9d630c754c5f8979c595b1f0a2e0 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/a/377a9d630c754c5f8979c595b1f0a2e0 2024-11-18T23:35:37,432 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/a/377a9d630c754c5f8979c595b1f0a2e0, entries=754, sequenceid=754, filesize=49.3 K 2024-11-18T23:35:37,433 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 83a955c880a10d3118e39eb8be5b22ee in 125ms, sequenceid=754, compaction requested=false; wal=null 2024-11-18T23:35:37,450 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-18T23:35:37,451 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 83a955c880a10d3118e39eb8be5b22ee 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-18T23:35:37,461 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/a/587b835bf51148e1a7c0c4e6e690a462 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731972931881/Put/seqid=0 2024-11-18T23:35:37,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741859_1036 (size=20072) 2024-11-18T23:35:37,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741859_1036 (size=20072) 2024-11-18T23:35:37,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741859_1036 (size=20072) 2024-11-18T23:35:37,504 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/a/587b835bf51148e1a7c0c4e6e690a462 2024-11-18T23:35:37,546 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/b/4376981c00a742c5b7e079111da87da1 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731972931923/Put/seqid=0 2024-11-18T23:35:37,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741860_1037 (size=35835) 2024-11-18T23:35:37,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741860_1037 (size=35835) 2024-11-18T23:35:37,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741860_1037 (size=35835) 2024-11-18T23:35:37,564 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/b/4376981c00a742c5b7e079111da87da1 2024-11-18T23:35:37,571 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/a/587b835bf51148e1a7c0c4e6e690a462 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/a/587b835bf51148e1a7c0c4e6e690a462 2024-11-18T23:35:37,579 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/a/587b835bf51148e1a7c0c4e6e690a462, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-18T23:35:37,581 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/b/4376981c00a742c5b7e079111da87da1 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/b/4376981c00a742c5b7e079111da87da1 2024-11-18T23:35:37,589 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/b/4376981c00a742c5b7e079111da87da1, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-18T23:35:37,589 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 83a955c880a10d3118e39eb8be5b22ee in 139ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-18T23:35:37,605 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-18T23:35:37,605 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 83a955c880a10d3118e39eb8be5b22ee 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-18T23:35:37,614 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/b/7d9f8d0757774577ad652b809c5bd272 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731972931956/Put/seqid=0 2024-11-18T23:35:37,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741861_1038 (size=35082) 2024-11-18T23:35:37,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741861_1038 (size=35082) 2024-11-18T23:35:37,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741861_1038 (size=35082) 2024-11-18T23:35:38,040 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/b/7d9f8d0757774577ad652b809c5bd272 2024-11-18T23:35:38,073 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/c/22b99bf2aee84a62b56f2fa19b0c79ef is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731972931998/Put/seqid=0 2024-11-18T23:35:38,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741862_1039 (size=20825) 2024-11-18T23:35:38,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741862_1039 (size=20825) 2024-11-18T23:35:38,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741862_1039 (size=20825) 2024-11-18T23:35:38,122 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/c/22b99bf2aee84a62b56f2fa19b0c79ef 2024-11-18T23:35:38,141 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/b/7d9f8d0757774577ad652b809c5bd272 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/b/7d9f8d0757774577ad652b809c5bd272 2024-11-18T23:35:38,162 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/b/7d9f8d0757774577ad652b809c5bd272, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-18T23:35:38,169 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/c/22b99bf2aee84a62b56f2fa19b0c79ef as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/c/22b99bf2aee84a62b56f2fa19b0c79ef 2024-11-18T23:35:38,223 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/c/22b99bf2aee84a62b56f2fa19b0c79ef, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-18T23:35:38,224 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 83a955c880a10d3118e39eb8be5b22ee in 619ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-18T23:35:38,238 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731972932073/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. 2024-11-18T23:35:38,243 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000003002 2024-11-18T23:35:38,244 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-18T23:35:38,248 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 83a955c880a10d3118e39eb8be5b22ee 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-18T23:35:38,281 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/c/2bffdcac3b7447b2a4047b6689a392b8 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731972932010/Put/seqid=0 2024-11-18T23:35:38,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741863_1040 (size=50301) 2024-11-18T23:35:38,339 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/c/2bffdcac3b7447b2a4047b6689a392b8 2024-11-18T23:35:38,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741863_1040 (size=50301) 2024-11-18T23:35:38,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741863_1040 (size=50301) 2024-11-18T23:35:38,356 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2bffdcac3b7447b2a4047b6689a392b8 2024-11-18T23:35:38,358 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/.tmp/c/2bffdcac3b7447b2a4047b6689a392b8 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/c/2bffdcac3b7447b2a4047b6689a392b8 2024-11-18T23:35:38,371 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2bffdcac3b7447b2a4047b6689a392b8 2024-11-18T23:35:38,372 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/c/2bffdcac3b7447b2a4047b6689a392b8, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-18T23:35:38,372 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 83a955c880a10d3118e39eb8be5b22ee in 128ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-18T23:35:38,373 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/0000000000000003002 2024-11-18T23:35:38,375 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:38,375 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:38,376 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:35:38,378 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 83a955c880a10d3118e39eb8be5b22ee 2024-11-18T23:35:38,385 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenIntoWAL/83a955c880a10d3118e39eb8be5b22ee/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-18T23:35:38,386 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 83a955c880a10d3118e39eb8be5b22ee; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60794880, jitterRate=-0.094085693359375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:35:38,387 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 83a955c880a10d3118e39eb8be5b22ee: Writing region info on filesystem at 1731972936698Initializing all the Stores at 1731972936699 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972936699Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972936700 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972936700Cleaning up temporary data from old regions at 1731972938375 (+1675 ms)Region opened successfully at 1731972938386 (+11 ms) 2024-11-18T23:35:38,475 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 83a955c880a10d3118e39eb8be5b22ee, disabling compactions & flushes 2024-11-18T23:35:38,475 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. 2024-11-18T23:35:38,475 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. 2024-11-18T23:35:38,475 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. after waiting 0 ms 2024-11-18T23:35:38,475 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. 2024-11-18T23:35:38,488 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731972931633.83a955c880a10d3118e39eb8be5b22ee. 2024-11-18T23:35:38,488 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 83a955c880a10d3118e39eb8be5b22ee: Waiting for close lock at 1731972938474Disabling compacts and flushes for region at 1731972938474Disabling writes for close at 1731972938475 (+1 ms)Writing region close event to WAL at 1731972938488 (+13 ms)Closed at 1731972938488 2024-11-18T23:35:38,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741857_1034 (size=93) 2024-11-18T23:35:38,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741857_1034 (size=93) 2024-11-18T23:35:38,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741857_1034 (size=93) 2024-11-18T23:35:38,501 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:35:38,501 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731972936652) 2024-11-18T23:35:38,522 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 387) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1498563925_22 at /127.0.0.1:59010 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:33425 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33425 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1498563925_22 at /127.0.0.1:58960 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:40073 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1498563925_22 at /127.0.0.1:45062 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1498563925_22 at /127.0.0.1:45336 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:37995 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40073 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:37995 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=861 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1502 (was 1545), ProcessCount=11 (was 11), AvailableMemoryMB=4689 (was 5378) 2024-11-18T23:35:38,541 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=861, MaxFileDescriptor=1048576, SystemLoadAverage=1502, ProcessCount=11, AvailableMemoryMB=4688 2024-11-18T23:35:38,559 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:38,562 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:38,563 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:38,570 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-78708374, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/hregion-78708374, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:38,586 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-78708374/hregion-78708374.1731972938570, exclude list is [], retry=0 2024-11-18T23:35:38,592 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:38,597 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:38,605 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:38,618 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-78708374/hregion-78708374.1731972938570 2024-11-18T23:35:38,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157)] 2024-11-18T23:35:38,620 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 94a991c37eaee5b3f80817b1a7a6900a, NAME => 'test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/hbase 2024-11-18T23:35:38,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741865_1042 (size=43) 2024-11-18T23:35:38,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741865_1042 (size=43) 2024-11-18T23:35:38,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741865_1042 (size=43) 2024-11-18T23:35:39,107 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:39,109 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:39,112 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94a991c37eaee5b3f80817b1a7a6900a columnFamilyName a 2024-11-18T23:35:39,112 DEBUG [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:39,113 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(327): Store=94a991c37eaee5b3f80817b1a7a6900a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:39,113 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:39,117 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94a991c37eaee5b3f80817b1a7a6900a columnFamilyName b 2024-11-18T23:35:39,117 DEBUG [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:39,118 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(327): Store=94a991c37eaee5b3f80817b1a7a6900a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:39,118 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:39,120 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94a991c37eaee5b3f80817b1a7a6900a columnFamilyName c 2024-11-18T23:35:39,120 DEBUG [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:39,121 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(327): Store=94a991c37eaee5b3f80817b1a7a6900a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:39,121 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:39,123 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:39,124 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:39,127 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:39,127 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:39,128 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:39,131 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:39,134 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:39,135 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 94a991c37eaee5b3f80817b1a7a6900a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61234791, jitterRate=-0.08753050863742828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:39,136 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 94a991c37eaee5b3f80817b1a7a6900a: Writing region info on filesystem at 1731972939108Initializing all the Stores at 1731972939109 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972939109Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972939109Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972939109Cleaning up temporary data from old regions at 1731972939127 (+18 ms)Region opened successfully at 1731972939136 (+9 ms) 2024-11-18T23:35:39,137 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 94a991c37eaee5b3f80817b1a7a6900a, disabling compactions & flushes 2024-11-18T23:35:39,137 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. 2024-11-18T23:35:39,137 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. 2024-11-18T23:35:39,137 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. after waiting 0 ms 2024-11-18T23:35:39,137 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. 2024-11-18T23:35:39,138 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. 2024-11-18T23:35:39,138 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 94a991c37eaee5b3f80817b1a7a6900a: Waiting for close lock at 1731972939137Disabling compacts and flushes for region at 1731972939137Disabling writes for close at 1731972939137Writing region close event to WAL at 1731972939138 (+1 ms)Closed at 1731972939138 2024-11-18T23:35:39,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741864_1041 (size=93) 2024-11-18T23:35:39,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741864_1041 (size=93) 2024-11-18T23:35:39,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741864_1041 (size=93) 2024-11-18T23:35:39,155 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:35:39,155 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-78708374:(num 1731972938570) 2024-11-18T23:35:39,156 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:39,159 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:39,179 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159, exclude list is [], retry=0 2024-11-18T23:35:39,185 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:39,186 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:39,186 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:39,200 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159 2024-11-18T23:35:39,205 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:35:39,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741866_1043 (size=344306) 2024-11-18T23:35:39,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741866_1043 (size=344306) 2024-11-18T23:35:39,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741866_1043 (size=344306) 2024-11-18T23:35:39,506 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159, size=336.2 K (344306bytes) 2024-11-18T23:35:39,506 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159 2024-11-18T23:35:39,506 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159 after 0ms 2024-11-18T23:35:39,513 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159 took 8ms 2024-11-18T23:35:39,531 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731972939159.temp 2024-11-18T23:35:39,537 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000000001-wal.1731972939159.temp 2024-11-18T23:35:39,562 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159 so closing down 2024-11-18T23:35:39,562 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:39,565 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:39,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741867_1044 (size=344306) 2024-11-18T23:35:39,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741867_1044 (size=344306) 2024-11-18T23:35:39,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741867_1044 (size=344306) 2024-11-18T23:35:39,589 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000000001-wal.1731972939159.temp (wrote 3000 edits, skipped 0 edits in 12 ms) 2024-11-18T23:35:39,592 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000000001-wal.1731972939159.temp to hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003000 2024-11-18T23:35:39,592 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-18T23:35:39,593 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159, journal: Splitting hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159, size=336.2 K (344306bytes) at 1731972939506Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000000001-wal.1731972939159.temp at 1731972939537 (+31 ms)Split 1024 edits, skipped 0 edits. at 1731972939538 (+1 ms)Split 2048 edits, skipped 0 edits. at 1731972939551 (+13 ms)Finishing writing output for hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159 so closing down at 1731972939562 (+11 ms)3 split writer threads finished at 1731972939565 (+3 ms)Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000000001-wal.1731972939159.temp (wrote 3000 edits, skipped 0 edits in 12 ms) at 1731972939590 (+25 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000000001-wal.1731972939159.temp to hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003000 at 1731972939592 (+2 ms)Processed 3000 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1731972939592 2024-11-18T23:35:39,595 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939159 to hdfs://localhost:37995/hbase/oldWALs/wal.1731972939159 2024-11-18T23:35:39,597 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003000 2024-11-18T23:35:39,597 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:39,600 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:39,619 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601, exclude list is [], retry=0 2024-11-18T23:35:39,626 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:39,626 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:39,627 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:39,640 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601 2024-11-18T23:35:39,640 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:35:39,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741868_1045 (size=344433) 2024-11-18T23:35:39,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741868_1045 (size=344433) 2024-11-18T23:35:39,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741868_1045 (size=344433) 2024-11-18T23:35:39,922 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601, size=336.4 K (344433bytes) 2024-11-18T23:35:39,922 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601 2024-11-18T23:35:39,923 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601 after 1ms 2024-11-18T23:35:39,928 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601 took 6ms 2024-11-18T23:35:39,948 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731972939601.temp 2024-11-18T23:35:39,957 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601 so closing down 2024-11-18T23:35:39,957 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003001-wal.1731972939601.temp 2024-11-18T23:35:39,957 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:39,969 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:40,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741869_1046 (size=344433) 2024-11-18T23:35:40,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741869_1046 (size=344433) 2024-11-18T23:35:40,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741869_1046 (size=344433) 2024-11-18T23:35:40,041 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003001-wal.1731972939601.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-18T23:35:40,047 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003001-wal.1731972939601.temp to hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000006000 2024-11-18T23:35:40,047 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 114 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-18T23:35:40,048 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601, journal: Splitting hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601, size=336.4 K (344433bytes) at 1731972939922Split 1024 edits, skipped 0 edits. at 1731972939942 (+20 ms)Split 2048 edits, skipped 0 edits. at 1731972939949 (+7 ms)Finishing writing output for hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601 so closing down at 1731972939957 (+8 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003001-wal.1731972939601.temp at 17319729399573 split writer threads finished at 1731972939969 (+12 ms)Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003001-wal.1731972939601.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1731972940042 (+73 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003001-wal.1731972939601.temp to hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000006000 at 1731972940047 (+5 ms)Processed 3000 edits across 1 Regions in 114 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1731972940047 2024-11-18T23:35:40,051 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972939601 to hdfs://localhost:37995/hbase/oldWALs/wal.1731972939601 2024-11-18T23:35:40,053 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000006000 2024-11-18T23:35:40,053 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:40,057 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/test2727-manual,16010,1731972938558, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:40,071 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972940057, exclude list is [], retry=0 2024-11-18T23:35:40,073 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:40,074 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:40,074 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:40,081 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731972938558/wal.1731972940057 2024-11-18T23:35:40,082 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:35:40,082 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 94a991c37eaee5b3f80817b1a7a6900a, NAME => 'test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:40,082 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:40,082 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,082 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,087 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,089 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94a991c37eaee5b3f80817b1a7a6900a columnFamilyName a 2024-11-18T23:35:40,089 DEBUG [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:40,090 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(327): Store=94a991c37eaee5b3f80817b1a7a6900a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:40,090 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,098 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94a991c37eaee5b3f80817b1a7a6900a columnFamilyName b 2024-11-18T23:35:40,098 DEBUG [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:40,099 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(327): Store=94a991c37eaee5b3f80817b1a7a6900a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:40,100 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,101 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94a991c37eaee5b3f80817b1a7a6900a columnFamilyName c 2024-11-18T23:35:40,101 DEBUG [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:40,101 INFO [StoreOpener-94a991c37eaee5b3f80817b1a7a6900a-1 {}] regionserver.HStore(327): Store=94a991c37eaee5b3f80817b1a7a6900a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:40,102 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,102 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,105 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,106 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003000 2024-11-18T23:35:40,144 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003000 2024-11-18T23:35:40,147 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000006000 2024-11-18T23:35:40,184 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000006000 2024-11-18T23:35:40,185 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 94a991c37eaee5b3f80817b1a7a6900a 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-18T23:35:40,210 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/.tmp/a/9b473922c60a4cbdaaeeb24a0199c2f1 is 41, key is test2727/a:100/1731972939646/Put/seqid=0 2024-11-18T23:35:40,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741871_1048 (size=84227) 2024-11-18T23:35:40,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741871_1048 (size=84227) 2024-11-18T23:35:40,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741871_1048 (size=84227) 2024-11-18T23:35:40,228 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/.tmp/a/9b473922c60a4cbdaaeeb24a0199c2f1 2024-11-18T23:35:40,262 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/.tmp/b/6264a61e328b418ab2d2d11f07876e12 is 41, key is test2727/b:100/1731972939709/Put/seqid=0 2024-11-18T23:35:40,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741872_1049 (size=84609) 2024-11-18T23:35:40,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741872_1049 (size=84609) 2024-11-18T23:35:40,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741872_1049 (size=84609) 2024-11-18T23:35:40,277 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/.tmp/b/6264a61e328b418ab2d2d11f07876e12 2024-11-18T23:35:40,308 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/.tmp/c/5b99d8904b7d4ba78155682dcc220d1c is 41, key is test2727/c:100/1731972939794/Put/seqid=0 2024-11-18T23:35:40,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741873_1050 (size=84609) 2024-11-18T23:35:40,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741873_1050 (size=84609) 2024-11-18T23:35:40,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741873_1050 (size=84609) 2024-11-18T23:35:40,321 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/.tmp/c/5b99d8904b7d4ba78155682dcc220d1c 2024-11-18T23:35:40,329 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/.tmp/a/9b473922c60a4cbdaaeeb24a0199c2f1 as hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/a/9b473922c60a4cbdaaeeb24a0199c2f1 2024-11-18T23:35:40,345 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/a/9b473922c60a4cbdaaeeb24a0199c2f1, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-18T23:35:40,346 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/.tmp/b/6264a61e328b418ab2d2d11f07876e12 as hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/b/6264a61e328b418ab2d2d11f07876e12 2024-11-18T23:35:40,354 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/b/6264a61e328b418ab2d2d11f07876e12, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-18T23:35:40,355 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/.tmp/c/5b99d8904b7d4ba78155682dcc220d1c as hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/c/5b99d8904b7d4ba78155682dcc220d1c 2024-11-18T23:35:40,363 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/c/5b99d8904b7d4ba78155682dcc220d1c, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-18T23:35:40,363 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 94a991c37eaee5b3f80817b1a7a6900a in 179ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-18T23:35:40,364 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000003000 2024-11-18T23:35:40,365 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/0000000000000006000 2024-11-18T23:35:40,366 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,366 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,367 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:40,369 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 94a991c37eaee5b3f80817b1a7a6900a 2024-11-18T23:35:40,371 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/test2727/94a991c37eaee5b3f80817b1a7a6900a/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-18T23:35:40,372 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 94a991c37eaee5b3f80817b1a7a6900a; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59550282, jitterRate=-0.1126316487789154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:40,373 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 94a991c37eaee5b3f80817b1a7a6900a: Writing region info on filesystem at 1731972940082Initializing all the Stores at 1731972940087 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972940087Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972940087Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972940087Obtaining lock to block concurrent updates at 1731972940185 (+98 ms)Preparing flush snapshotting stores in 94a991c37eaee5b3f80817b1a7a6900a at 1731972940185Finished memstore snapshotting test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731972940185Flushing stores of test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. at 1731972940185Flushing 94a991c37eaee5b3f80817b1a7a6900a/a: creating writer at 1731972940185Flushing 94a991c37eaee5b3f80817b1a7a6900a/a: appending metadata at 1731972940209 (+24 ms)Flushing 94a991c37eaee5b3f80817b1a7a6900a/a: closing flushed file at 1731972940209Flushing 94a991c37eaee5b3f80817b1a7a6900a/b: creating writer at 1731972940234 (+25 ms)Flushing 94a991c37eaee5b3f80817b1a7a6900a/b: appending metadata at 1731972940260 (+26 ms)Flushing 94a991c37eaee5b3f80817b1a7a6900a/b: closing flushed file at 1731972940260Flushing 94a991c37eaee5b3f80817b1a7a6900a/c: creating writer at 1731972940285 (+25 ms)Flushing 94a991c37eaee5b3f80817b1a7a6900a/c: appending metadata at 1731972940308 (+23 ms)Flushing 94a991c37eaee5b3f80817b1a7a6900a/c: closing flushed file at 1731972940308Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@791a4412: reopening flushed file at 1731972940328 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66f2b0ea: reopening flushed file at 1731972940345 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f434f28: reopening flushed file at 1731972940354 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 94a991c37eaee5b3f80817b1a7a6900a in 179ms, sequenceid=6000, compaction requested=false; wal=null at 1731972940363 (+9 ms)Cleaning up temporary data from old regions at 1731972940367 (+4 ms)Region opened successfully at 1731972940373 (+6 ms) 2024-11-18T23:35:40,374 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-18T23:35:40,375 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 94a991c37eaee5b3f80817b1a7a6900a, disabling compactions & flushes 2024-11-18T23:35:40,375 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. 2024-11-18T23:35:40,375 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. 2024-11-18T23:35:40,375 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. after waiting 0 ms 2024-11-18T23:35:40,375 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. 2024-11-18T23:35:40,376 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731972938560.94a991c37eaee5b3f80817b1a7a6900a. 2024-11-18T23:35:40,376 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 94a991c37eaee5b3f80817b1a7a6900a: Waiting for close lock at 1731972940375Disabling compacts and flushes for region at 1731972940375Disabling writes for close at 1731972940375Writing region close event to WAL at 1731972940376 (+1 ms)Closed at 1731972940376 2024-11-18T23:35:40,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741870_1047 (size=93) 2024-11-18T23:35:40,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741870_1047 (size=93) 2024-11-18T23:35:40,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741870_1047 (size=93) 2024-11-18T23:35:40,384 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:35:40,384 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731972940057) 2024-11-18T23:35:40,402 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=399 (was 399), OpenFileDescriptor=921 (was 861) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1502 (was 1502), ProcessCount=11 (was 11), AvailableMemoryMB=4525 (was 4688) 2024-11-18T23:35:40,416 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=400, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=1502, ProcessCount=11, AvailableMemoryMB=4524 2024-11-18T23:35:40,435 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:40,445 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:40,446 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731972940446 2024-11-18T23:35:40,462 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446 2024-11-18T23:35:40,469 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:35:40,473 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 438c11ecc4317b93cf3be0a31e994604, NAME => 'testSequentialEditLogSeqNum,,1731972940436.438c11ecc4317b93cf3be0a31e994604.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:40,473 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731972940436.438c11ecc4317b93cf3be0a31e994604.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:40,473 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 438c11ecc4317b93cf3be0a31e994604 2024-11-18T23:35:40,473 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 438c11ecc4317b93cf3be0a31e994604 2024-11-18T23:35:40,474 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604 doesn't exist for region: 438c11ecc4317b93cf3be0a31e994604 on table testSequentialEditLogSeqNum 2024-11-18T23:35:40,475 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 438c11ecc4317b93cf3be0a31e994604 on table testSequentialEditLogSeqNum 2024-11-18T23:35:40,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741875_1052 (size=62) 2024-11-18T23:35:40,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741875_1052 (size=62) 2024-11-18T23:35:40,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741875_1052 (size=62) 2024-11-18T23:35:40,504 INFO [StoreOpener-438c11ecc4317b93cf3be0a31e994604-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 438c11ecc4317b93cf3be0a31e994604 2024-11-18T23:35:40,506 INFO [StoreOpener-438c11ecc4317b93cf3be0a31e994604-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 438c11ecc4317b93cf3be0a31e994604 columnFamilyName a 2024-11-18T23:35:40,506 DEBUG [StoreOpener-438c11ecc4317b93cf3be0a31e994604-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:40,507 INFO [StoreOpener-438c11ecc4317b93cf3be0a31e994604-1 {}] regionserver.HStore(327): Store=438c11ecc4317b93cf3be0a31e994604/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:40,507 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 438c11ecc4317b93cf3be0a31e994604 2024-11-18T23:35:40,508 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604 2024-11-18T23:35:40,508 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604 2024-11-18T23:35:40,509 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 438c11ecc4317b93cf3be0a31e994604 2024-11-18T23:35:40,509 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 438c11ecc4317b93cf3be0a31e994604 2024-11-18T23:35:40,513 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 438c11ecc4317b93cf3be0a31e994604 2024-11-18T23:35:40,518 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:40,519 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 438c11ecc4317b93cf3be0a31e994604; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67464984, jitterRate=0.005306601524353027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:35:40,520 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 438c11ecc4317b93cf3be0a31e994604: Writing region info on filesystem at 1731972940474Initializing all the Stores at 1731972940501 (+27 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972940501Cleaning up temporary data from old regions at 1731972940509 (+8 ms)Region opened successfully at 1731972940520 (+11 ms) 2024-11-18T23:35:40,534 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 438c11ecc4317b93cf3be0a31e994604 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-18T23:35:40,566 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/.tmp/a/f95146e2cc4341e0b8eca4cd227a19b6 is 81, key is testSequentialEditLogSeqNum/a:x0/1731972940520/Put/seqid=0 2024-11-18T23:35:40,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741876_1053 (size=5833) 2024-11-18T23:35:40,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741876_1053 (size=5833) 2024-11-18T23:35:40,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741876_1053 (size=5833) 2024-11-18T23:35:40,581 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/.tmp/a/f95146e2cc4341e0b8eca4cd227a19b6 2024-11-18T23:35:40,592 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/.tmp/a/f95146e2cc4341e0b8eca4cd227a19b6 as hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/a/f95146e2cc4341e0b8eca4cd227a19b6 2024-11-18T23:35:40,607 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/a/f95146e2cc4341e0b8eca4cd227a19b6, entries=10, sequenceid=13, filesize=5.7 K 2024-11-18T23:35:40,609 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 438c11ecc4317b93cf3be0a31e994604 in 75ms, sequenceid=13, compaction requested=false 2024-11-18T23:35:40,610 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 438c11ecc4317b93cf3be0a31e994604: 2024-11-18T23:35:40,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-18T23:35:40,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-18T23:35:40,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-18T23:35:40,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-18T23:35:40,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-18T23:35:40,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741874_1051 (size=3017) 2024-11-18T23:35:40,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741874_1051 (size=3017) 2024-11-18T23:35:40,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741874_1051 (size=3017) 2024-11-18T23:35:41,044 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446, size=2.9 K (3017bytes) 2024-11-18T23:35:41,044 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446 2024-11-18T23:35:41,044 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446 after 0ms 2024-11-18T23:35:41,047 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446 took 4ms 2024-11-18T23:35:41,049 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446 so closing down 2024-11-18T23:35:41,049 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:41,050 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731972940446.temp 2024-11-18T23:35:41,052 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/recovered.edits/0000000000000000003-wal.1731972940446.temp 2024-11-18T23:35:41,052 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:41,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741877_1054 (size=2433) 2024-11-18T23:35:41,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741877_1054 (size=2433) 2024-11-18T23:35:41,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741877_1054 (size=2433) 2024-11-18T23:35:41,062 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/recovered.edits/0000000000000000003-wal.1731972940446.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-18T23:35:41,063 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/recovered.edits/0000000000000000003-wal.1731972940446.temp to hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/recovered.edits/0000000000000000020 2024-11-18T23:35:41,064 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-18T23:35:41,064 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446, journal: Splitting hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446, size=2.9 K (3017bytes) at 1731972941044Finishing writing output for hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446 so closing down at 1731972941049 (+5 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/recovered.edits/0000000000000000003-wal.1731972940446.temp at 1731972941052 (+3 ms)3 split writer threads finished at 1731972941052Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/recovered.edits/0000000000000000003-wal.1731972940446.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731972941062 (+10 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/recovered.edits/0000000000000000003-wal.1731972940446.temp to hdfs://localhost:37995/hbase/data/default/testSequentialEditLogSeqNum/438c11ecc4317b93cf3be0a31e994604/recovered.edits/0000000000000000020 at 1731972941064 (+2 ms)Processed 17 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:37995/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731972940434/wal.1731972940446, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1731972941064 2024-11-18T23:35:41,077 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=406 (was 400) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:59010 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:45492 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:45168 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1502 (was 1502), ProcessCount=11 (was 11), AvailableMemoryMB=4518 (was 4524) 2024-11-18T23:35:41,088 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=406, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=1502, ProcessCount=11, AvailableMemoryMB=4517 2024-11-18T23:35:41,105 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:41,108 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:41,141 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:41,145 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38825733, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/hregion-38825733, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:41,166 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38825733/hregion-38825733.1731972941145, exclude list is [], retry=0 2024-11-18T23:35:41,172 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:41,172 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:41,177 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:41,185 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38825733/hregion-38825733.1731972941145 2024-11-18T23:35:41,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:35:41,185 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4a1854680a53c46362aaa52a6bbe8a44, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/hbase 2024-11-18T23:35:41,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741879_1056 (size=70) 2024-11-18T23:35:41,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741879_1056 (size=70) 2024-11-18T23:35:41,203 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:41,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741879_1056 (size=70) 2024-11-18T23:35:41,212 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,214 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1854680a53c46362aaa52a6bbe8a44 columnFamilyName a 2024-11-18T23:35:41,214 DEBUG [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:41,215 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(327): Store=4a1854680a53c46362aaa52a6bbe8a44/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:41,215 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,218 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1854680a53c46362aaa52a6bbe8a44 columnFamilyName b 2024-11-18T23:35:41,218 DEBUG [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:41,219 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(327): Store=4a1854680a53c46362aaa52a6bbe8a44/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:41,219 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,222 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1854680a53c46362aaa52a6bbe8a44 columnFamilyName c 2024-11-18T23:35:41,222 DEBUG [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:41,223 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(327): Store=4a1854680a53c46362aaa52a6bbe8a44/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:41,223 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,225 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,225 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,226 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:41,227 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,233 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:41,234 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4a1854680a53c46362aaa52a6bbe8a44; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69216800, jitterRate=0.03141069412231445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:41,234 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4a1854680a53c46362aaa52a6bbe8a44: Writing region info on filesystem at 1731972941203Initializing all the Stores at 1731972941211 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972941211Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972941211Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972941211Cleaning up temporary data from old regions at 1731972941225 (+14 ms)Region opened successfully at 1731972941234 (+9 ms) 2024-11-18T23:35:41,235 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4a1854680a53c46362aaa52a6bbe8a44, disabling compactions & flushes 2024-11-18T23:35:41,235 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. 2024-11-18T23:35:41,235 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. 2024-11-18T23:35:41,235 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. after waiting 0 ms 2024-11-18T23:35:41,235 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. 2024-11-18T23:35:41,235 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. 2024-11-18T23:35:41,236 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4a1854680a53c46362aaa52a6bbe8a44: Waiting for close lock at 1731972941235Disabling compacts and flushes for region at 1731972941235Disabling writes for close at 1731972941235Writing region close event to WAL at 1731972941235Closed at 1731972941235 2024-11-18T23:35:41,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741878_1055 (size=93) 2024-11-18T23:35:41,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741878_1055 (size=93) 2024-11-18T23:35:41,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741878_1055 (size=93) 2024-11-18T23:35:41,250 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:35:41,250 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38825733:(num 1731972941145) 2024-11-18T23:35:41,251 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:41,257 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:41,271 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257, exclude list is [], retry=0 2024-11-18T23:35:41,275 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:41,275 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:41,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:41,285 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 2024-11-18T23:35:41,285 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:35:41,286 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a1854680a53c46362aaa52a6bbe8a44, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:41,286 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:41,286 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,286 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,295 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,297 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1854680a53c46362aaa52a6bbe8a44 columnFamilyName a 2024-11-18T23:35:41,297 DEBUG [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:41,298 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(327): Store=4a1854680a53c46362aaa52a6bbe8a44/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:41,298 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,299 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1854680a53c46362aaa52a6bbe8a44 columnFamilyName b 2024-11-18T23:35:41,299 DEBUG [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:41,300 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(327): Store=4a1854680a53c46362aaa52a6bbe8a44/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:41,300 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,302 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1854680a53c46362aaa52a6bbe8a44 columnFamilyName c 2024-11-18T23:35:41,302 DEBUG [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:41,305 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(327): Store=4a1854680a53c46362aaa52a6bbe8a44/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:41,305 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,306 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,307 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,310 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,310 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,311 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:41,313 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:41,314 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4a1854680a53c46362aaa52a6bbe8a44; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72533382, jitterRate=0.0808316171169281}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:41,315 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4a1854680a53c46362aaa52a6bbe8a44: Writing region info on filesystem at 1731972941286Initializing all the Stores at 1731972941292 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972941293 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972941295 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972941295Cleaning up temporary data from old regions at 1731972941310 (+15 ms)Region opened successfully at 1731972941315 (+5 ms) 2024-11-18T23:35:41,322 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731972941321/Put/seqid=0 2024-11-18T23:35:41,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741881_1058 (size=4826) 2024-11-18T23:35:41,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741881_1058 (size=4826) 2024-11-18T23:35:41,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741881_1058 (size=4826) 2024-11-18T23:35:41,347 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37995/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 4a1854680a53c46362aaa52a6bbe8a44/a 2024-11-18T23:35:41,356 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-18T23:35:41,356 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-18T23:35:41,356 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4a1854680a53c46362aaa52a6bbe8a44: 2024-11-18T23:35:41,358 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/a/347bab28ebbb4990890757dc70aac2bd_SeqId_3_ 2024-11-18T23:35:41,359 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37995/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 4a1854680a53c46362aaa52a6bbe8a44/a as hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/a/347bab28ebbb4990890757dc70aac2bd_SeqId_3_ - updating store file list. 2024-11-18T23:35:41,365 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 347bab28ebbb4990890757dc70aac2bd_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-18T23:35:41,366 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/a/347bab28ebbb4990890757dc70aac2bd_SeqId_3_ into 4a1854680a53c46362aaa52a6bbe8a44/a 2024-11-18T23:35:41,366 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37995/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 4a1854680a53c46362aaa52a6bbe8a44/a (new location: hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/a/347bab28ebbb4990890757dc70aac2bd_SeqId_3_) 2024-11-18T23:35:41,409 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257, size=0 (0bytes) 2024-11-18T23:35:41,409 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 might be still open, length is 0 2024-11-18T23:35:41,409 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 2024-11-18T23:35:41,410 WARN [IPC Server handler 4 on default port 37995 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-18T23:35:41,410 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 after 1ms 2024-11-18T23:35:43,402 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:45244 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45244 dst: /127.0.0.1:36973 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36973 remote=/127.0.0.1:45244]. Total timeout mills is 60000, 57969 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:43,403 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:59184 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39707:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59184 dst: /127.0.0.1:39707 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:43,403 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:45584 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35345:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45584 dst: /127.0.0.1:35345 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:43,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741880_1059 (size=504) 2024-11-18T23:35:43,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741880_1059 (size=504) 2024-11-18T23:35:43,916 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-18T23:35:43,916 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-18T23:35:43,918 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-18T23:35:43,918 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-18T23:35:45,411 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 after 4002ms 2024-11-18T23:35:45,416 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 took 4007ms 2024-11-18T23:35:45,430 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257; continuing. 2024-11-18T23:35:45,430 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 so closing down 2024-11-18T23:35:45,430 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:45,439 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731972941257.temp 2024-11-18T23:35:45,455 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005-wal.1731972941257.temp 2024-11-18T23:35:45,456 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:45,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741882_1060 (size=238) 2024-11-18T23:35:45,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741882_1060 (size=238) 2024-11-18T23:35:45,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741882_1060 (size=238) 2024-11-18T23:35:45,536 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005-wal.1731972941257.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-18T23:35:45,539 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005-wal.1731972941257.temp to hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005 2024-11-18T23:35:45,540 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 114 ms; skipped=1; WAL=hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257, size=0, length=0, corrupted=false, cancelled=false 2024-11-18T23:35:45,540 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257, journal: Splitting hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257, size=0 (0bytes) at 1731972941409Finishing writing output for hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 so closing down at 1731972945430 (+4021 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005-wal.1731972941257.temp at 1731972945455 (+25 ms)3 split writer threads finished at 1731972945456 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005-wal.1731972941257.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731972945536 (+80 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005-wal.1731972941257.temp to hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005 at 1731972945539 (+3 ms)Processed 2 edits across 1 Regions in 114 ms; skipped=1; WAL=hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257, size=0, length=0, corrupted=false, cancelled=false at 1731972945540 (+1 ms) 2024-11-18T23:35:45,544 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 to hdfs://localhost:37995/hbase/oldWALs/wal.1731972941257 2024-11-18T23:35:45,548 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005 2024-11-18T23:35:45,548 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:45,565 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:45,588 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972945565, exclude list is [], retry=0 2024-11-18T23:35:45,598 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:45,598 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:45,598 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:45,633 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972945565 2024-11-18T23:35:45,635 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:42157:42157)] 2024-11-18T23:35:45,636 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a1854680a53c46362aaa52a6bbe8a44, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:45,636 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:45,636 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,636 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,643 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,649 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1854680a53c46362aaa52a6bbe8a44 columnFamilyName a 2024-11-18T23:35:45,649 DEBUG [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:45,683 DEBUG [StoreFileOpener-4a1854680a53c46362aaa52a6bbe8a44-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 347bab28ebbb4990890757dc70aac2bd_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-18T23:35:45,683 DEBUG [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/a/347bab28ebbb4990890757dc70aac2bd_SeqId_3_ 2024-11-18T23:35:45,683 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(327): Store=4a1854680a53c46362aaa52a6bbe8a44/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:45,683 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,686 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1854680a53c46362aaa52a6bbe8a44 columnFamilyName b 2024-11-18T23:35:45,686 DEBUG [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:45,689 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(327): Store=4a1854680a53c46362aaa52a6bbe8a44/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:45,689 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,691 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a1854680a53c46362aaa52a6bbe8a44 columnFamilyName c 2024-11-18T23:35:45,691 DEBUG [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:45,692 INFO [StoreOpener-4a1854680a53c46362aaa52a6bbe8a44-1 {}] regionserver.HStore(327): Store=4a1854680a53c46362aaa52a6bbe8a44/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:45,692 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,695 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,701 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005 2024-11-18T23:35:45,715 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005 2024-11-18T23:35:45,715 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4a1854680a53c46362aaa52a6bbe8a44 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-18T23:35:45,743 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/.tmp/a/cb792f3cf37e44c4b212c72de0360aa7 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731972941371/Put/seqid=0 2024-11-18T23:35:45,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741884_1062 (size=5149) 2024-11-18T23:35:45,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741884_1062 (size=5149) 2024-11-18T23:35:45,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741884_1062 (size=5149) 2024-11-18T23:35:45,769 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/.tmp/a/cb792f3cf37e44c4b212c72de0360aa7 2024-11-18T23:35:45,787 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/.tmp/a/cb792f3cf37e44c4b212c72de0360aa7 as hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/a/cb792f3cf37e44c4b212c72de0360aa7 2024-11-18T23:35:45,803 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/a/cb792f3cf37e44c4b212c72de0360aa7, entries=1, sequenceid=5, filesize=5.0 K 2024-11-18T23:35:45,804 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 4a1854680a53c46362aaa52a6bbe8a44 in 89ms, sequenceid=5, compaction requested=false; wal=null 2024-11-18T23:35:45,807 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/0000000000000000005 2024-11-18T23:35:45,812 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,812 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,820 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:45,833 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4a1854680a53c46362aaa52a6bbe8a44 2024-11-18T23:35:45,844 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4a1854680a53c46362aaa52a6bbe8a44/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-18T23:35:45,846 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4a1854680a53c46362aaa52a6bbe8a44; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60596128, jitterRate=-0.09704732894897461}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:45,846 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4a1854680a53c46362aaa52a6bbe8a44: Writing region info on filesystem at 1731972945637Initializing all the Stores at 1731972945643 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972945643Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972945643Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972945643Obtaining lock to block concurrent updates at 1731972945715 (+72 ms)Preparing flush snapshotting stores in 4a1854680a53c46362aaa52a6bbe8a44 at 1731972945715Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731972945716 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. at 1731972945716Flushing 4a1854680a53c46362aaa52a6bbe8a44/a: creating writer at 1731972945716Flushing 4a1854680a53c46362aaa52a6bbe8a44/a: appending metadata at 1731972945742 (+26 ms)Flushing 4a1854680a53c46362aaa52a6bbe8a44/a: closing flushed file at 1731972945742Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18355160: reopening flushed file at 1731972945783 (+41 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 4a1854680a53c46362aaa52a6bbe8a44 in 89ms, sequenceid=5, compaction requested=false; wal=null at 1731972945804 (+21 ms)Cleaning up temporary data from old regions at 1731972945812 (+8 ms)Region opened successfully at 1731972945846 (+34 ms) 2024-11-18T23:35:45,852 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4a1854680a53c46362aaa52a6bbe8a44, disabling compactions & flushes 2024-11-18T23:35:45,853 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. 2024-11-18T23:35:45,853 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. 2024-11-18T23:35:45,853 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. after waiting 0 ms 2024-11-18T23:35:45,853 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. 2024-11-18T23:35:45,862 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731972941106.4a1854680a53c46362aaa52a6bbe8a44. 2024-11-18T23:35:45,862 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4a1854680a53c46362aaa52a6bbe8a44: Waiting for close lock at 1731972945852Disabling compacts and flushes for region at 1731972945852Disabling writes for close at 1731972945853 (+1 ms)Writing region close event to WAL at 1731972945862 (+9 ms)Closed at 1731972945862 2024-11-18T23:35:45,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741883_1061 (size=93) 2024-11-18T23:35:45,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741883_1061 (size=93) 2024-11-18T23:35:45,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741883_1061 (size=93) 2024-11-18T23:35:45,886 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:35:45,886 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731972945565) 2024-11-18T23:35:45,901 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 406) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:37995 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2136071527_22 at /127.0.0.1:55612 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:37995 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2136071527_22 at /127.0.0.1:42916 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1477 (was 1502), ProcessCount=11 (was 11), AvailableMemoryMB=6086 (was 4517) - AvailableMemoryMB LEAK? - 2024-11-18T23:35:45,916 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=1477, ProcessCount=11, AvailableMemoryMB=6082 2024-11-18T23:35:45,935 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:45,940 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:35:45,945 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d96596e55a9,40599,1731972923486 2024-11-18T23:35:45,949 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a08bb48 2024-11-18T23:35:45,950 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:35:45,952 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:35:45,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:35:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-18T23:35:45,972 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T23:35:45,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-18T23:35:45,974 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:45,976 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T23:35:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:35:46,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741885_1063 (size=694) 2024-11-18T23:35:46,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741885_1063 (size=694) 2024-11-18T23:35:46,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741885_1063 (size=694) 2024-11-18T23:35:46,012 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b890df8ea0d9342b29ca822fc6dfed52, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b 2024-11-18T23:35:46,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741886_1064 (size=77) 2024-11-18T23:35:46,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741886_1064 (size=77) 2024-11-18T23:35:46,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741886_1064 (size=77) 2024-11-18T23:35:46,027 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:46,027 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing b890df8ea0d9342b29ca822fc6dfed52, disabling compactions & flushes 2024-11-18T23:35:46,027 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,027 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,028 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. after waiting 0 ms 2024-11-18T23:35:46,028 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,028 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,028 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for b890df8ea0d9342b29ca822fc6dfed52: Waiting for close lock at 1731972946027Disabling compacts and flushes for region at 1731972946027Disabling writes for close at 1731972946028 (+1 ms)Writing region close event to WAL at 1731972946028Closed at 1731972946028 2024-11-18T23:35:46,031 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T23:35:46,036 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731972946031"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731972946031"}]},"ts":"1731972946031"} 2024-11-18T23:35:46,042 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T23:35:46,048 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T23:35:46,051 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731972946048"}]},"ts":"1731972946048"} 2024-11-18T23:35:46,064 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-18T23:35:46,065 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9d96596e55a9=0} racks are {/default-rack=0} 2024-11-18T23:35:46,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T23:35:46,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T23:35:46,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-18T23:35:46,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T23:35:46,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T23:35:46,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-18T23:35:46,073 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T23:35:46,073 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T23:35:46,073 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-18T23:35:46,073 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T23:35:46,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, ASSIGN}] 2024-11-18T23:35:46,088 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, ASSIGN 2024-11-18T23:35:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:35:46,100 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, ASSIGN; state=OFFLINE, location=9d96596e55a9,46105,1731972924571; forceNewPlan=false, retain=false 2024-11-18T23:35:46,255 INFO [9d96596e55a9:40599 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T23:35:46,256 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=OPENING, regionLocation=9d96596e55a9,46105,1731972924571 2024-11-18T23:35:46,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, ASSIGN because future has completed 2024-11-18T23:35:46,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46105,1731972924571}] 2024-11-18T23:35:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:35:46,439 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:35:46,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51939, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:35:46,447 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,447 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b890df8ea0d9342b29ca822fc6dfed52, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:46,448 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,448 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:46,448 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,448 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,449 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,451 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf1 2024-11-18T23:35:46,451 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:46,452 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:46,452 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,454 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf2 2024-11-18T23:35:46,454 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:46,454 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:46,455 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,456 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,457 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,457 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,457 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,459 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-18T23:35:46,460 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,463 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:46,464 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b890df8ea0d9342b29ca822fc6dfed52; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68901179, jitterRate=0.026707574725151062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-18T23:35:46,464 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,464 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b890df8ea0d9342b29ca822fc6dfed52: Running coprocessor pre-open hook at 1731972946448Writing region info on filesystem at 1731972946448Initializing all the Stores at 1731972946449 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972946449Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972946449Cleaning up temporary data from old regions at 1731972946458 (+9 ms)Running coprocessor post-open hooks at 1731972946464 (+6 ms)Region opened successfully at 1731972946464 2024-11-18T23:35:46,466 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., pid=6, masterSystemTime=1731972946439 2024-11-18T23:35:46,469 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,469 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,470 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=OPEN, openSeqNum=2, regionLocation=9d96596e55a9,46105,1731972924571 2024-11-18T23:35:46,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46105,1731972924571 because future has completed 2024-11-18T23:35:46,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T23:35:46,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46105,1731972924571 in 212 msec 2024-11-18T23:35:46,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T23:35:46,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, ASSIGN in 408 msec 2024-11-18T23:35:46,490 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T23:35:46,490 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731972946490"}]},"ts":"1731972946490"} 2024-11-18T23:35:46,497 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-18T23:35:46,499 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T23:35:46,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 541 msec 2024-11-18T23:35:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:35:46,609 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-18T23:35:46,609 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-18T23:35:46,610 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T23:35:46,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-18T23:35:46,617 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T23:35:46,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-18T23:35:46,634 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=2] 2024-11-18T23:35:46,636 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:35:46,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:35:46,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=b890df8ea0d9342b29ca822fc6dfed52, source=9d96596e55a9,46105,1731972924571, destination=9d96596e55a9,33655,1731972924943, warming up region on 9d96596e55a9,33655,1731972924943 2024-11-18T23:35:46,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=b890df8ea0d9342b29ca822fc6dfed52, source=9d96596e55a9,46105,1731972924571, destination=9d96596e55a9,33655,1731972924943, running balancer 2024-11-18T23:35:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE 2024-11-18T23:35:46,659 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE 2024-11-18T23:35:46,663 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=CLOSING, regionLocation=9d96596e55a9,46105,1731972924571 2024-11-18T23:35:46,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE because future has completed 2024-11-18T23:35:46,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] regionserver.HRegion(7855): Warmup {ENCODED => b890df8ea0d9342b29ca822fc6dfed52, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:46,677 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,679 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf1 2024-11-18T23:35:46,679 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:46,680 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:46,680 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,680 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:35:46,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46105,1731972924571}] 2024-11-18T23:35:46,690 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf2 2024-11-18T23:35:46,690 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:46,692 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] regionserver.HRegion(1722): Closing b890df8ea0d9342b29ca822fc6dfed52, disabling compactions & flushes 2024-11-18T23:35:46,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. after waiting 0 ms 2024-11-18T23:35:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-18T23:35:46,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] regionserver.HRegion(1676): Region close journal for b890df8ea0d9342b29ca822fc6dfed52: Waiting for close lock at 1731972946693Disabling compacts and flushes for region at 1731972946693Disabling writes for close at 1731972946694 (+1 ms)Writing region close event to WAL at 1731972946712 (+18 ms)Closed at 1731972946712 2024-11-18T23:35:46,841 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,841 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:35:46,842 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing b890df8ea0d9342b29ca822fc6dfed52, disabling compactions & flushes 2024-11-18T23:35:46,842 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,842 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,842 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. after waiting 0 ms 2024-11-18T23:35:46,842 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,842 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing b890df8ea0d9342b29ca822fc6dfed52 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-18T23:35:46,858 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf1/78470dbb6eb9477297a9fd6c360f06b5 is 35, key is r1/cf1:q/1731972946639/Put/seqid=0 2024-11-18T23:35:46,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741887_1065 (size=4783) 2024-11-18T23:35:46,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741887_1065 (size=4783) 2024-11-18T23:35:46,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741887_1065 (size=4783) 2024-11-18T23:35:46,870 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf1/78470dbb6eb9477297a9fd6c360f06b5 2024-11-18T23:35:46,879 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf1/78470dbb6eb9477297a9fd6c360f06b5 as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/78470dbb6eb9477297a9fd6c360f06b5 2024-11-18T23:35:46,886 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/78470dbb6eb9477297a9fd6c360f06b5, entries=1, sequenceid=5, filesize=4.7 K 2024-11-18T23:35:46,888 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for b890df8ea0d9342b29ca822fc6dfed52 in 46ms, sequenceid=5, compaction requested=false 2024-11-18T23:35:46,888 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-18T23:35:46,897 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T23:35:46,900 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:46,900 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for b890df8ea0d9342b29ca822fc6dfed52: Waiting for close lock at 1731972946842Running coprocessor pre-close hooks at 1731972946842Disabling compacts and flushes for region at 1731972946842Disabling writes for close at 1731972946842Obtaining lock to block concurrent updates at 1731972946842Preparing flush snapshotting stores in b890df8ea0d9342b29ca822fc6dfed52 at 1731972946842Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731972946842Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. at 1731972946843 (+1 ms)Flushing b890df8ea0d9342b29ca822fc6dfed52/cf1: creating writer at 1731972946843Flushing b890df8ea0d9342b29ca822fc6dfed52/cf1: appending metadata at 1731972946858 (+15 ms)Flushing b890df8ea0d9342b29ca822fc6dfed52/cf1: closing flushed file at 1731972946858Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b856a94: reopening flushed file at 1731972946878 (+20 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for b890df8ea0d9342b29ca822fc6dfed52 in 46ms, sequenceid=5, compaction requested=false at 1731972946888 (+10 ms)Writing region close event to WAL at 1731972946889 (+1 ms)Running coprocessor post-close hooks at 1731972946898 (+9 ms)Closed at 1731972946900 (+2 ms) 2024-11-18T23:35:46,901 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding b890df8ea0d9342b29ca822fc6dfed52 move to 9d96596e55a9,33655,1731972924943 record at close sequenceid=5 2024-11-18T23:35:46,909 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:46,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=CLOSED 2024-11-18T23:35:46,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46105,1731972924571 because future has completed 2024-11-18T23:35:46,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T23:35:46,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46105,1731972924571 in 242 msec 2024-11-18T23:35:46,929 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE; state=CLOSED, location=9d96596e55a9,33655,1731972924943; forceNewPlan=false, retain=false 2024-11-18T23:35:47,080 INFO [9d96596e55a9:40599 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T23:35:47,080 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=OPENING, regionLocation=9d96596e55a9,33655,1731972924943 2024-11-18T23:35:47,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE because future has completed 2024-11-18T23:35:47,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,33655,1731972924943}] 2024-11-18T23:35:47,243 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:47,243 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => b890df8ea0d9342b29ca822fc6dfed52, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:47,243 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,243 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:47,244 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,244 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,252 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,254 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf1 2024-11-18T23:35:47,254 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:47,267 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/78470dbb6eb9477297a9fd6c360f06b5 2024-11-18T23:35:47,267 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:47,267 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,268 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf2 2024-11-18T23:35:47,268 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:47,270 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:47,271 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,272 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,274 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,275 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,276 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,277 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-18T23:35:47,279 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,281 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened b890df8ea0d9342b29ca822fc6dfed52; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70105651, jitterRate=0.044655606150627136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-18T23:35:47,281 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:47,282 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for b890df8ea0d9342b29ca822fc6dfed52: Running coprocessor pre-open hook at 1731972947244Writing region info on filesystem at 1731972947244Initializing all the Stores at 1731972947246 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972947246Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972947252 (+6 ms)Cleaning up temporary data from old regions at 1731972947276 (+24 ms)Running coprocessor post-open hooks at 1731972947281 (+5 ms)Region opened successfully at 1731972947282 (+1 ms) 2024-11-18T23:35:47,283 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., pid=9, masterSystemTime=1731972947238 2024-11-18T23:35:47,292 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:47,293 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:47,293 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=OPEN, openSeqNum=9, regionLocation=9d96596e55a9,33655,1731972924943 2024-11-18T23:35:47,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,33655,1731972924943 because future has completed 2024-11-18T23:35:47,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-18T23:35:47,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,33655,1731972924943 in 219 msec 2024-11-18T23:35:47,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE in 658 msec 2024-11-18T23:35:47,365 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:35:47,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:35:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:33556 deadline: 1731973007371, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9d96596e55a9 port=33655 startCode=1731972924943. As of locationSeqNum=5. 2024-11-18T23:35:47,377 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9d96596e55a9 port=33655 startCode=1731972924943. As of locationSeqNum=5. 2024-11-18T23:35:47,378 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9d96596e55a9 port=33655 startCode=1731972924943. As of locationSeqNum=5. 2024-11-18T23:35:47,378 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,33655,1731972924943, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9d96596e55a9 port=33655 startCode=1731972924943. As of locationSeqNum=5. 2024-11-18T23:35:47,509 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b890df8ea0d9342b29ca822fc6dfed52 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-18T23:35:47,550 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf1/e0ff2137cc0543ee87d6305b75d9ef1e is 29, key is r1/cf1:/1731972947489/DeleteFamily/seqid=0 2024-11-18T23:35:47,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741888_1066 (size=4906) 2024-11-18T23:35:47,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741888_1066 (size=4906) 2024-11-18T23:35:47,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741888_1066 (size=4906) 2024-11-18T23:35:47,573 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf1/e0ff2137cc0543ee87d6305b75d9ef1e 2024-11-18T23:35:47,593 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0ff2137cc0543ee87d6305b75d9ef1e 2024-11-18T23:35:47,619 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf2/26b7473919274e9a9470b7f01b261847 is 29, key is r1/cf2:/1731972947489/DeleteFamily/seqid=0 2024-11-18T23:35:47,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741889_1067 (size=4906) 2024-11-18T23:35:47,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741889_1067 (size=4906) 2024-11-18T23:35:47,718 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf2/26b7473919274e9a9470b7f01b261847 2024-11-18T23:35:47,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741889_1067 (size=4906) 2024-11-18T23:35:47,734 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 26b7473919274e9a9470b7f01b261847 2024-11-18T23:35:47,736 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf1/e0ff2137cc0543ee87d6305b75d9ef1e as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/e0ff2137cc0543ee87d6305b75d9ef1e 2024-11-18T23:35:47,746 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0ff2137cc0543ee87d6305b75d9ef1e 2024-11-18T23:35:47,746 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/e0ff2137cc0543ee87d6305b75d9ef1e, entries=1, sequenceid=12, filesize=4.8 K 2024-11-18T23:35:47,748 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf2/26b7473919274e9a9470b7f01b261847 as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/26b7473919274e9a9470b7f01b261847 2024-11-18T23:35:47,755 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 26b7473919274e9a9470b7f01b261847 2024-11-18T23:35:47,756 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/26b7473919274e9a9470b7f01b261847, entries=1, sequenceid=12, filesize=4.8 K 2024-11-18T23:35:47,762 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for b890df8ea0d9342b29ca822fc6dfed52 in 254ms, sequenceid=12, compaction requested=false 2024-11-18T23:35:47,762 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b890df8ea0d9342b29ca822fc6dfed52: 2024-11-18T23:35:47,765 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T23:35:47,768 DEBUG [Time-limited test {}] regionserver.HStore(1541): b890df8ea0d9342b29ca822fc6dfed52/cf1 is initiating major compaction (all files) 2024-11-18T23:35:47,776 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:35:47,776 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:47,777 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of b890df8ea0d9342b29ca822fc6dfed52/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:47,778 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/78470dbb6eb9477297a9fd6c360f06b5, hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/e0ff2137cc0543ee87d6305b75d9ef1e] into tmpdir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp, totalSize=9.5 K 2024-11-18T23:35:47,780 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 78470dbb6eb9477297a9fd6c360f06b5, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731972946639 2024-11-18T23:35:47,782 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e0ff2137cc0543ee87d6305b75d9ef1e, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-18T23:35:47,801 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): b890df8ea0d9342b29ca822fc6dfed52#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:35:47,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741890_1068 (size=4626) 2024-11-18T23:35:47,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741890_1068 (size=4626) 2024-11-18T23:35:47,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741890_1068 (size=4626) 2024-11-18T23:35:47,924 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf1/f8ce2f36881e4a84a3d39a885dd231fa as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/f8ce2f36881e4a84a3d39a885dd231fa 2024-11-18T23:35:47,980 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in b890df8ea0d9342b29ca822fc6dfed52/cf1 of b890df8ea0d9342b29ca822fc6dfed52 into f8ce2f36881e4a84a3d39a885dd231fa(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:35:47,980 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for b890df8ea0d9342b29ca822fc6dfed52: 2024-11-18T23:35:47,981 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-18T23:35:47,981 DEBUG [Time-limited test {}] regionserver.HStore(1541): b890df8ea0d9342b29ca822fc6dfed52/cf2 is initiating major compaction (all files) 2024-11-18T23:35:47,981 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:35:47,981 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:35:47,982 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of b890df8ea0d9342b29ca822fc6dfed52/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:47,982 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/26b7473919274e9a9470b7f01b261847] into tmpdir=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp, totalSize=4.8 K 2024-11-18T23:35:47,983 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 26b7473919274e9a9470b7f01b261847, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-18T23:35:47,998 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): b890df8ea0d9342b29ca822fc6dfed52#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:35:48,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741891_1069 (size=4592) 2024-11-18T23:35:48,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741891_1069 (size=4592) 2024-11-18T23:35:48,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741891_1069 (size=4592) 2024-11-18T23:35:48,055 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/.tmp/cf2/a3bb3e9fb5494d0bbf3cc44914c34381 as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/a3bb3e9fb5494d0bbf3cc44914c34381 2024-11-18T23:35:48,069 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in b890df8ea0d9342b29ca822fc6dfed52/cf2 of b890df8ea0d9342b29ca822fc6dfed52 into a3bb3e9fb5494d0bbf3cc44914c34381(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:35:48,069 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for b890df8ea0d9342b29ca822fc6dfed52: 2024-11-18T23:35:48,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=b890df8ea0d9342b29ca822fc6dfed52, source=9d96596e55a9,33655,1731972924943, destination=9d96596e55a9,46105,1731972924571, warming up region on 9d96596e55a9,46105,1731972924571 2024-11-18T23:35:48,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=b890df8ea0d9342b29ca822fc6dfed52, source=9d96596e55a9,33655,1731972924943, destination=9d96596e55a9,46105,1731972924571, running balancer 2024-11-18T23:35:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE 2024-11-18T23:35:48,080 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE 2024-11-18T23:35:48,082 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=CLOSING, regionLocation=9d96596e55a9,33655,1731972924943 2024-11-18T23:35:48,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE because future has completed 2024-11-18T23:35:48,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(7855): Warmup {ENCODED => b890df8ea0d9342b29ca822fc6dfed52, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:48,089 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,091 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf1 2024-11-18T23:35:48,091 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:48,092 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:35:48,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,33655,1731972924943}] 2024-11-18T23:35:48,124 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/78470dbb6eb9477297a9fd6c360f06b5 2024-11-18T23:35:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-18T23:35:48,134 INFO [StoreFileOpener-b890df8ea0d9342b29ca822fc6dfed52-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0ff2137cc0543ee87d6305b75d9ef1e 2024-11-18T23:35:48,135 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/e0ff2137cc0543ee87d6305b75d9ef1e 2024-11-18T23:35:48,158 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/f8ce2f36881e4a84a3d39a885dd231fa 2024-11-18T23:35:48,159 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:48,159 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,161 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf2 2024-11-18T23:35:48,161 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:48,200 INFO [StoreFileOpener-b890df8ea0d9342b29ca822fc6dfed52-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 26b7473919274e9a9470b7f01b261847 2024-11-18T23:35:48,200 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/26b7473919274e9a9470b7f01b261847 2024-11-18T23:35:48,226 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/a3bb3e9fb5494d0bbf3cc44914c34381 2024-11-18T23:35:48,226 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(1722): Closing b890df8ea0d9342b29ca822fc6dfed52, disabling compactions & flushes 2024-11-18T23:35:48,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. after waiting 0 ms 2024-11-18T23:35:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46105 {}] regionserver.HRegion(1676): Region close journal for b890df8ea0d9342b29ca822fc6dfed52: Waiting for close lock at 1731972948228Disabling compacts and flushes for region at 1731972948228Disabling writes for close at 1731972948229 (+1 ms)Writing region close event to WAL at 1731972948236 (+7 ms)Closed at 1731972948236 2024-11-18T23:35:48,249 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,249 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:35:48,249 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing b890df8ea0d9342b29ca822fc6dfed52, disabling compactions & flushes 2024-11-18T23:35:48,249 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,249 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,250 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. after waiting 0 ms 2024-11-18T23:35:48,250 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,254 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/78470dbb6eb9477297a9fd6c360f06b5, hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/e0ff2137cc0543ee87d6305b75d9ef1e] to archive 2024-11-18T23:35:48,258 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:35:48,271 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/78470dbb6eb9477297a9fd6c360f06b5 to hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/78470dbb6eb9477297a9fd6c360f06b5 2024-11-18T23:35:48,278 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/e0ff2137cc0543ee87d6305b75d9ef1e to hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/e0ff2137cc0543ee87d6305b75d9ef1e 2024-11-18T23:35:48,297 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/26b7473919274e9a9470b7f01b261847] to archive 2024-11-18T23:35:48,300 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:35:48,305 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/26b7473919274e9a9470b7f01b261847 to hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/26b7473919274e9a9470b7f01b261847 2024-11-18T23:35:48,334 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-18T23:35:48,336 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,336 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for b890df8ea0d9342b29ca822fc6dfed52: Waiting for close lock at 1731972948249Running coprocessor pre-close hooks at 1731972948249Disabling compacts and flushes for region at 1731972948249Disabling writes for close at 1731972948250 (+1 ms)Writing region close event to WAL at 1731972948316 (+66 ms)Running coprocessor post-close hooks at 1731972948335 (+19 ms)Closed at 1731972948336 (+1 ms) 2024-11-18T23:35:48,336 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding b890df8ea0d9342b29ca822fc6dfed52 move to 9d96596e55a9,46105,1731972924571 record at close sequenceid=12 2024-11-18T23:35:48,343 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=CLOSED 2024-11-18T23:35:48,343 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,33655,1731972924943 because future has completed 2024-11-18T23:35:48,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-18T23:35:48,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,33655,1731972924943 in 280 msec 2024-11-18T23:35:48,381 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE; state=CLOSED, location=9d96596e55a9,46105,1731972924571; forceNewPlan=false, retain=false 2024-11-18T23:35:48,532 INFO [9d96596e55a9:40599 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T23:35:48,533 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=OPENING, regionLocation=9d96596e55a9,46105,1731972924571 2024-11-18T23:35:48,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE because future has completed 2024-11-18T23:35:48,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46105,1731972924571}] 2024-11-18T23:35:48,697 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,697 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => b890df8ea0d9342b29ca822fc6dfed52, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:48,698 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,698 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:48,698 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,698 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,700 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,702 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf1 2024-11-18T23:35:48,702 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:48,711 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/f8ce2f36881e4a84a3d39a885dd231fa 2024-11-18T23:35:48,712 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:48,712 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,713 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf2 2024-11-18T23:35:48,714 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:48,732 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/a3bb3e9fb5494d0bbf3cc44914c34381 2024-11-18T23:35:48,732 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:48,732 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,733 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,735 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,736 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,736 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,738 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-18T23:35:48,741 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,743 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened b890df8ea0d9342b29ca822fc6dfed52; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59503151, jitterRate=-0.11333395540714264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-18T23:35:48,743 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,744 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for b890df8ea0d9342b29ca822fc6dfed52: Running coprocessor pre-open hook at 1731972948699Writing region info on filesystem at 1731972948699Initializing all the Stores at 1731972948700 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972948700Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972948700Cleaning up temporary data from old regions at 1731972948736 (+36 ms)Running coprocessor post-open hooks at 1731972948743 (+7 ms)Region opened successfully at 1731972948744 (+1 ms) 2024-11-18T23:35:48,745 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., pid=12, masterSystemTime=1731972948691 2024-11-18T23:35:48,749 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,749 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,750 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=OPEN, openSeqNum=18, regionLocation=9d96596e55a9,46105,1731972924571 2024-11-18T23:35:48,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46105,1731972924571 because future has completed 2024-11-18T23:35:48,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-18T23:35:48,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46105,1731972924571 in 219 msec 2024-11-18T23:35:48,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, REOPEN/MOVE in 683 msec 2024-11-18T23:35:48,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:35:48,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:35:48,791 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 9d96596e55a9,46105,1731972924571: testing ***** 2024-11-18T23:35:48,792 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-18T23:35:48,794 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-18T23:35:48,795 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-18T23:35:48,802 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-18T23:35:48,805 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-18T23:35:48,821 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 519143256 }, "NonHeapMemoryUsage": { "committed": 170786816, "init": 7667712, "max": -1, "used": 168117064 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9d96596e55a9", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 156, "ProcessCallTime_mean": 10, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 5, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 55, "ProcessCallTime_98th_percentile": 156, "ProcessCallTime_99th_percentile": 156, "ProcessCallTime_99.9th_percentile": 156, "ProcessCallTime_TimeRangeCount_0-1": 7, "ProcessCallTime_TimeRangeCount_1-3": 11, "ProcessCallTime_TimeRangeCount_3-10": 10, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "ProcessCallTime_TimeRangeCount_100-300": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 156, "TotalCallTime_mean": 10, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 5, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 55, "TotalCallTime_98th_percentile": 156, "TotalCallTime_99th_percentile": 156, "TotalCallTime_99.9th_percentile": 156, "TotalCallTime_TimeRangeCount_0-1": 6, "TotalCallTime_TimeRangeCount_1-3": 12, "TotalCallTime_TimeRangeCount_3-10": 10, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "TotalCallTime_TimeRangeCount_100-300": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 160, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5782 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9d96596e55a9", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59409", "tag.serverName": "9d96596e55a9,46105,1731972924571", "tag.clusterId": "0eef78ff-7039-4105-a286-74bff0b7861a", "tag.Context": "regionserver", "tag.Hostname": "9d96596e55a9", "regionCount": 1, "storeCount": 0, "hlogFileCount": 4, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731972924571, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.7697053155307054, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 4, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 1, "Put_min": 2, "Put_max": 2, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_1-3": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 46, "FlushTime_max": 46, "FlushTime_mean": 46, "FlushTime_25th_percentile": 46, "FlushTime_median": 46, "FlushTime_75th_percentile": 46, "FlushTime_90th_percentile": 46, "FlushTime_95th_percentile": 46, "FlushTime_98th_percentile": 46, "FlushTime_99th_percentile": 46, "FlushTime_99.9th_percentile": 46, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 64, "ScanSize_max": 64, "ScanSize_mean": 64, "ScanSize_25th_percentile": 64, "ScanSize_median": 64, "ScanSize_75th_percentile": 64, "ScanSize_90th_percentile": 64, "ScanSize_95th_percentile": 64, "ScanSize_98th_percentile": 64, "ScanSize_99th_percentile": 64, "ScanSize_99.9th_percentile": 64, "ScanSize_SizeRangeCount_10-100": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-18T23:35:48,829 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40599 {}] master.MasterRpcServices(700): 9d96596e55a9,46105,1731972924571 reported a fatal error: ***** ABORTING region server 9d96596e55a9,46105,1731972924571: testing ***** 2024-11-18T23:35:48,837 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9d96596e55a9,46105,1731972924571' ***** 2024-11-18T23:35:48,837 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-18T23:35:48,837 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T23:35:48,838 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T23:35:48,838 INFO [RS:0;9d96596e55a9:46105 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-18T23:35:48,838 INFO [RS:0;9d96596e55a9:46105 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-18T23:35:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33655 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:42334 deadline: 1731973008838, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9d96596e55a9 port=46105 startCode=1731972924571. As of locationSeqNum=12. 2024-11-18T23:35:48,839 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(3091): Received CLOSE for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,839 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(956): aborting server 9d96596e55a9,46105,1731972924571 2024-11-18T23:35:48,839 INFO [RS:0;9d96596e55a9:46105 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:35:48,839 INFO [RS:0;9d96596e55a9:46105 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9d96596e55a9:46105. 2024-11-18T23:35:48,840 DEBUG [RS:0;9d96596e55a9:46105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:35:48,840 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,33655,1731972924943, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,33655,1731972924943, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9d96596e55a9 port=46105 startCode=1731972924571. As of locationSeqNum=12. 2024-11-18T23:35:48,840 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,33655,1731972924943, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9d96596e55a9 port=46105 startCode=1731972924571. As of locationSeqNum=12. 2024-11-18T23:35:48,840 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,33655,1731972924943, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9d96596e55a9 port=46105 startCode=1731972924571. As of locationSeqNum=12. 2024-11-18T23:35:48,840 DEBUG [RS:0;9d96596e55a9:46105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:35:48,840 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b890df8ea0d9342b29ca822fc6dfed52, disabling compactions & flushes 2024-11-18T23:35:48,840 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,840 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,840 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. after waiting 0 ms 2024-11-18T23:35:48,840 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,841 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T23:35:48,841 DEBUG [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(1325): Online Regions={b890df8ea0d9342b29ca822fc6dfed52=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.} 2024-11-18T23:35:48,841 DEBUG [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(1351): Waiting on b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:48,845 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,845 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b890df8ea0d9342b29ca822fc6dfed52: Waiting for close lock at 1731972948839Running coprocessor pre-close hooks at 1731972948840 (+1 ms)Disabling compacts and flushes for region at 1731972948840Disabling writes for close at 1731972948840Writing region close event to WAL at 1731972948845 (+5 ms)Running coprocessor post-close hooks at 1731972948845Closed at 1731972948845 2024-11-18T23:35:48,845 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:48,860 INFO [regionserver/9d96596e55a9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:35:48,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9d96596e55a9,46105,1731972924571 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:48,950 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9d96596e55a9,46105,1731972924571 aborting 2024-11-18T23:35:48,950 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9d96596e55a9,46105,1731972924571 aborting 2024-11-18T23:35:48,950 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=12 from cache 2024-11-18T23:35:49,041 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(976): stopping server 9d96596e55a9,46105,1731972924571; all regions closed. 2024-11-18T23:35:49,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741835_1011 (size=2142) 2024-11-18T23:35:49,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741835_1011 (size=2142) 2024-11-18T23:35:49,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741835_1011 (size=2142) 2024-11-18T23:35:49,164 DEBUG [RS:0;9d96596e55a9:46105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:35:49,164 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:35:49,164 INFO [RS:0;9d96596e55a9:46105 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:35:49,164 INFO [RS:0;9d96596e55a9:46105 {}] hbase.ChoreService(370): Chore service for: regionserver/9d96596e55a9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T23:35:49,164 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18] 2024-11-18T23:35:49,165 INFO [regionserver/9d96596e55a9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:35:49,165 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T23:35:49,165 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T23:35:49,165 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T23:35:49,165 INFO [RS:0;9d96596e55a9:46105 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:35:49,165 INFO [RS:0;9d96596e55a9:46105 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46105 2024-11-18T23:35:49,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9d96596e55a9,46105,1731972924571 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:49,166 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9d96596e55a9,46105,1731972924571 aborting 2024-11-18T23:35:49,166 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9d96596e55a9,46105,1731972924571 aborting 2024-11-18T23:35:49,166 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18 from cache 2024-11-18T23:35:49,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9d96596e55a9,46105,1731972924571 2024-11-18T23:35:49,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:35:49,442 INFO [RS:0;9d96596e55a9:46105 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:35:49,444 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9d96596e55a9,46105,1731972924571] 2024-11-18T23:35:49,482 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18] 2024-11-18T23:35:49,484 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9d96596e55a9:46105 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9d96596e55a9/172.17.0.2:46105 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:49,485 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18, error=java.net.ConnectException: Call to address=9d96596e55a9:46105 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9d96596e55a9/172.17.0.2:46105 2024-11-18T23:35:49,485 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-18T23:35:49,485 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18 from cache 2024-11-18T23:35:49,485 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 9d96596e55a9:46105 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9d96596e55a9/172.17.0.2:46105 2024-11-18T23:35:49,586 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9d96596e55a9,46105,1731972924571 already deleted, retry=false 2024-11-18T23:35:49,586 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9d96596e55a9,46105,1731972924571 on 9d96596e55a9,40599,1731972923486 2024-11-18T23:35:49,593 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9d96596e55a9,46105,1731972924571, splitWal=true, meta=false 2024-11-18T23:35:49,595 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9d96596e55a9,46105,1731972924571, splitWal=true, meta=false 2024-11-18T23:35:49,595 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 9d96596e55a9,46105,1731972924571 (carryingMeta=false) 9d96596e55a9,46105,1731972924571/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@347dcd17[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-18T23:35:49,597 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 9d96596e55a9,46105,1731972924571 had 1 regions 2024-11-18T23:35:49,599 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9d96596e55a9,46105,1731972924571, splitWal=true, meta=false, isMeta: false 2024-11-18T23:35:49,601 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting 2024-11-18T23:35:49,602 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9d96596e55a9,46105,1731972924571 WAL count=1, meta=false 2024-11-18T23:35:49,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9d96596e55a9%2C46105%2C1731972924571.1731972927993}] 2024-11-18T23:35:49,611 DEBUG [PEWorker-5 {}] master.SplitWALManager(158): Acquired split WAL worker=9d96596e55a9,46043,1731972925008 2024-11-18T23:35:49,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9d96596e55a9%2C46105%2C1731972924571.1731972927993, worker=9d96596e55a9,46043,1731972925008}] 2024-11-18T23:35:49,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:35:49,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46105-0x101504d78350001, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:35:49,670 INFO [RS:0;9d96596e55a9:46105 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:35:49,670 INFO [RS:0;9d96596e55a9:46105 {}] regionserver.HRegionServer(1031): Exiting; stopping=9d96596e55a9,46105,1731972924571; zookeeper connection closed. 2024-11-18T23:35:49,670 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@66ae3a9a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@66ae3a9a 2024-11-18T23:35:49,776 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:35:49,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51949, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:35:49,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46043 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-18T23:35:49,806 INFO [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993, size=2.1 K (2142bytes) 2024-11-18T23:35:49,806 INFO [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993 2024-11-18T23:35:49,806 INFO [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993 after 0ms 2024-11-18T23:35:49,810 INFO [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993 took 4ms 2024-11-18T23:35:49,824 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for b890df8ea0d9342b29ca822fc6dfed52: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-18T23:35:49,824 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993 so closing down 2024-11-18T23:35:49,824 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:49,826 INFO [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:49,826 INFO [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 13 ms; skipped=6; WAL=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-18T23:35:49,826 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993, journal: Splitting hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993, size=2.1 K (2142bytes) at 1731972949806Finishing writing output for hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993 so closing down at 1731972949824 (+18 ms)3 split writer threads finished at 1731972949826 (+2 ms)Processed 6 edits across 0 Regions in 13 ms; skipped=6; WAL=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1731972949826 2024-11-18T23:35:49,826 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993 2024-11-18T23:35:49,828 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-18T23:35:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-18T23:35:49,842 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting/9d96596e55a9%2C46105%2C1731972924571.1731972927993 to hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/oldWALs 2024-11-18T23:35:49,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-18T23:35:49,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9d96596e55a9%2C46105%2C1731972924571.1731972927993, worker=9d96596e55a9,46043,1731972925008 in 228 msec 2024-11-18T23:35:49,847 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=9d96596e55a9,46043,1731972925008 2024-11-18T23:35:49,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-18T23:35:49,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 9d96596e55a9%2C46105%2C1731972924571.1731972927993, worker=9d96596e55a9,46043,1731972925008 in 244 msec 2024-11-18T23:35:49,854 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting dir is empty, no logs to split. 2024-11-18T23:35:49,854 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9d96596e55a9,46105,1731972924571 WAL count=0, meta=false 2024-11-18T23:35:49,854 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 9d96596e55a9,46105,1731972924571 WAL splitting is done? wals=0, meta=false 2024-11-18T23:35:49,856 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 9d96596e55a9,46105,1731972924571 failed, ignore...File hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,46105,1731972924571-splitting does not exist. 2024-11-18T23:35:49,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, ASSIGN}] 2024-11-18T23:35:49,861 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, ASSIGN 2024-11-18T23:35:49,862 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T23:35:50,001 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18] 2024-11-18T23:35:50,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9d96596e55a9:46105 this server is in the failed servers list 2024-11-18T23:35:50,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9d96596e55a9:46105 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9d96596e55a9:46105 2024-11-18T23:35:50,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9d96596e55a9:46105 2024-11-18T23:35:50,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46105,1731972924571, seqNum=18 from cache 2024-11-18T23:35:50,013 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(204): Hosts are {9d96596e55a9=0} racks are {/default-rack=0} 2024-11-18T23:35:50,014 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T23:35:50,014 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T23:35:50,014 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T23:35:50,014 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T23:35:50,014 INFO [9d96596e55a9:40599 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T23:35:50,014 INFO [9d96596e55a9:40599 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T23:35:50,014 DEBUG [9d96596e55a9:40599 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T23:35:50,014 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=OPENING, regionLocation=9d96596e55a9,46043,1731972925008 2024-11-18T23:35:50,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, ASSIGN because future has completed 2024-11-18T23:35:50,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46043,1731972925008}] 2024-11-18T23:35:50,175 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:50,175 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => b890df8ea0d9342b29ca822fc6dfed52, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:50,176 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,176 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:50,176 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,176 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,188 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,189 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf1 2024-11-18T23:35:50,189 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:50,202 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf1/f8ce2f36881e4a84a3d39a885dd231fa 2024-11-18T23:35:50,202 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:50,202 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,205 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b890df8ea0d9342b29ca822fc6dfed52 columnFamilyName cf2 2024-11-18T23:35:50,205 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:50,216 DEBUG [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/cf2/a3bb3e9fb5494d0bbf3cc44914c34381 2024-11-18T23:35:50,216 INFO [StoreOpener-b890df8ea0d9342b29ca822fc6dfed52-1 {}] regionserver.HStore(327): Store=b890df8ea0d9342b29ca822fc6dfed52/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:50,217 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,220 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,229 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,235 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,235 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,236 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-18T23:35:50,239 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,240 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened b890df8ea0d9342b29ca822fc6dfed52; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74550866, jitterRate=0.11089447140693665}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-18T23:35:50,241 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:35:50,242 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for b890df8ea0d9342b29ca822fc6dfed52: Running coprocessor pre-open hook at 1731972950176Writing region info on filesystem at 1731972950176Initializing all the Stores at 1731972950185 (+9 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972950185Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972950187 (+2 ms)Cleaning up temporary data from old regions at 1731972950235 (+48 ms)Running coprocessor post-open hooks at 1731972950241 (+6 ms)Region opened successfully at 1731972950241 2024-11-18T23:35:50,244 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., pid=17, masterSystemTime=1731972950171 2024-11-18T23:35:50,249 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:50,249 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:35:50,251 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=b890df8ea0d9342b29ca822fc6dfed52, regionState=OPEN, openSeqNum=18, regionLocation=9d96596e55a9,46043,1731972925008 2024-11-18T23:35:50,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46043,1731972925008 because future has completed 2024-11-18T23:35:50,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-18T23:35:50,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure b890df8ea0d9342b29ca822fc6dfed52, server=9d96596e55a9,46043,1731972925008 in 238 msec 2024-11-18T23:35:50,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-18T23:35:50,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b890df8ea0d9342b29ca822fc6dfed52, ASSIGN in 418 msec 2024-11-18T23:35:50,301 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 9d96596e55a9,46105,1731972924571 after splitting done 2024-11-18T23:35:50,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 9d96596e55a9,46105,1731972924571, splitWal=true, meta=false in 714 msec 2024-11-18T23:35:51,012 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52., hostname=9d96596e55a9,46043,1731972925008, seqNum=18] 2024-11-18T23:35:51,012 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:35:51,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:35:51,036 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 412), OpenFileDescriptor=1043 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1431 (was 1477), ProcessCount=11 (was 11), AvailableMemoryMB=5309 (was 6082) 2024-11-18T23:35:51,038 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1043 is superior to 1024 2024-11-18T23:35:51,059 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1043, MaxFileDescriptor=1048576, SystemLoadAverage=1431, ProcessCount=11, AvailableMemoryMB=5306 2024-11-18T23:35:51,059 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1043 is superior to 1024 2024-11-18T23:35:51,081 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:51,086 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:51,088 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:51,102 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-25988426, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/hregion-25988426, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:51,121 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-25988426/hregion-25988426.1731972951103, exclude list is [], retry=0 2024-11-18T23:35:51,128 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:51,128 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:51,129 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:51,147 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-25988426/hregion-25988426.1731972951103 2024-11-18T23:35:51,148 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:35:51,148 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3c61bff5d9fab9a0c88f2ec1a66f421b, NAME => 'testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/hbase 2024-11-18T23:35:51,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741893_1071 (size=67) 2024-11-18T23:35:51,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741893_1071 (size=67) 2024-11-18T23:35:51,165 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:51,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741893_1071 (size=67) 2024-11-18T23:35:51,168 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,169 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c61bff5d9fab9a0c88f2ec1a66f421b columnFamilyName a 2024-11-18T23:35:51,169 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:51,170 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(327): Store=3c61bff5d9fab9a0c88f2ec1a66f421b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:51,170 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,172 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c61bff5d9fab9a0c88f2ec1a66f421b columnFamilyName b 2024-11-18T23:35:51,172 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:51,172 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(327): Store=3c61bff5d9fab9a0c88f2ec1a66f421b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:51,173 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,174 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c61bff5d9fab9a0c88f2ec1a66f421b columnFamilyName c 2024-11-18T23:35:51,174 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:51,175 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(327): Store=3c61bff5d9fab9a0c88f2ec1a66f421b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:51,175 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,175 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,177 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,177 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,177 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:51,178 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,181 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:51,181 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3c61bff5d9fab9a0c88f2ec1a66f421b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67890893, jitterRate=0.011653140187263489}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:51,182 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3c61bff5d9fab9a0c88f2ec1a66f421b: Writing region info on filesystem at 1731972951166Initializing all the Stores at 1731972951166Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972951167 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972951167Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972951167Cleaning up temporary data from old regions at 1731972951177 (+10 ms)Region opened successfully at 1731972951182 (+5 ms) 2024-11-18T23:35:51,182 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3c61bff5d9fab9a0c88f2ec1a66f421b, disabling compactions & flushes 2024-11-18T23:35:51,182 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. 2024-11-18T23:35:51,182 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. 2024-11-18T23:35:51,182 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. after waiting 0 ms 2024-11-18T23:35:51,182 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. 2024-11-18T23:35:51,182 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. 2024-11-18T23:35:51,182 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3c61bff5d9fab9a0c88f2ec1a66f421b: Waiting for close lock at 1731972951182Disabling compacts and flushes for region at 1731972951182Disabling writes for close at 1731972951182Writing region close event to WAL at 1731972951182Closed at 1731972951182 2024-11-18T23:35:51,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741892_1070 (size=93) 2024-11-18T23:35:51,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741892_1070 (size=93) 2024-11-18T23:35:51,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741892_1070 (size=93) 2024-11-18T23:35:51,187 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:35:51,187 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-25988426:(num 1731972951103) 2024-11-18T23:35:51,187 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:51,189 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:51,200 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189, exclude list is [], retry=0 2024-11-18T23:35:51,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:51,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:51,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:51,204 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189 2024-11-18T23:35:51,205 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:35:51,205 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c61bff5d9fab9a0c88f2ec1a66f421b, NAME => 'testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:51,205 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:51,205 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,205 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,206 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,207 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c61bff5d9fab9a0c88f2ec1a66f421b columnFamilyName a 2024-11-18T23:35:51,207 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:51,208 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(327): Store=3c61bff5d9fab9a0c88f2ec1a66f421b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:51,208 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,208 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c61bff5d9fab9a0c88f2ec1a66f421b columnFamilyName b 2024-11-18T23:35:51,208 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:51,209 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(327): Store=3c61bff5d9fab9a0c88f2ec1a66f421b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:51,209 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,210 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c61bff5d9fab9a0c88f2ec1a66f421b columnFamilyName c 2024-11-18T23:35:51,210 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:51,210 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(327): Store=3c61bff5d9fab9a0c88f2ec1a66f421b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:51,210 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,213 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,214 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,214 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,215 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:51,216 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,218 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3c61bff5d9fab9a0c88f2ec1a66f421b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67453893, jitterRate=0.005141332745552063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:51,218 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3c61bff5d9fab9a0c88f2ec1a66f421b: Writing region info on filesystem at 1731972951205Initializing all the Stores at 1731972951206 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972951206Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972951206Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972951206Cleaning up temporary data from old regions at 1731972951214 (+8 ms)Region opened successfully at 1731972951218 (+4 ms) 2024-11-18T23:35:51,271 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3c61bff5d9fab9a0c88f2ec1a66f421b 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-18T23:35:51,288 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/a/5bbf1f35060a4ddd98f0dbd1619657ad is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731972951219/Put/seqid=0 2024-11-18T23:35:51,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741895_1073 (size=5958) 2024-11-18T23:35:51,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741895_1073 (size=5958) 2024-11-18T23:35:51,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741895_1073 (size=5958) 2024-11-18T23:35:51,317 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/a/5bbf1f35060a4ddd98f0dbd1619657ad 2024-11-18T23:35:51,350 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/b/6af4ad671e004fd8b51d7c866832b71f is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731972951239/Put/seqid=0 2024-11-18T23:35:51,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741896_1074 (size=5958) 2024-11-18T23:35:51,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741896_1074 (size=5958) 2024-11-18T23:35:51,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741896_1074 (size=5958) 2024-11-18T23:35:51,364 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/b/6af4ad671e004fd8b51d7c866832b71f 2024-11-18T23:35:51,392 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/c/50befa934d0048ef840eb66473d6db36 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731972951257/Put/seqid=0 2024-11-18T23:35:51,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741897_1075 (size=5958) 2024-11-18T23:35:51,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741897_1075 (size=5958) 2024-11-18T23:35:51,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741897_1075 (size=5958) 2024-11-18T23:35:51,411 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/c/50befa934d0048ef840eb66473d6db36 2024-11-18T23:35:51,419 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/a/5bbf1f35060a4ddd98f0dbd1619657ad as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/a/5bbf1f35060a4ddd98f0dbd1619657ad 2024-11-18T23:35:51,428 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/a/5bbf1f35060a4ddd98f0dbd1619657ad, entries=10, sequenceid=33, filesize=5.8 K 2024-11-18T23:35:51,429 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/b/6af4ad671e004fd8b51d7c866832b71f as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/b/6af4ad671e004fd8b51d7c866832b71f 2024-11-18T23:35:51,437 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/b/6af4ad671e004fd8b51d7c866832b71f, entries=10, sequenceid=33, filesize=5.8 K 2024-11-18T23:35:51,442 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/c/50befa934d0048ef840eb66473d6db36 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/c/50befa934d0048ef840eb66473d6db36 2024-11-18T23:35:51,450 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/c/50befa934d0048ef840eb66473d6db36, entries=10, sequenceid=33, filesize=5.8 K 2024-11-18T23:35:51,454 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 3c61bff5d9fab9a0c88f2ec1a66f421b in 183ms, sequenceid=33, compaction requested=false 2024-11-18T23:35:51,455 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3c61bff5d9fab9a0c88f2ec1a66f421b: 2024-11-18T23:35:51,455 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3c61bff5d9fab9a0c88f2ec1a66f421b, disabling compactions & flushes 2024-11-18T23:35:51,455 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. 2024-11-18T23:35:51,455 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. 2024-11-18T23:35:51,455 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. after waiting 0 ms 2024-11-18T23:35:51,455 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. 2024-11-18T23:35:51,471 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. 2024-11-18T23:35:51,471 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3c61bff5d9fab9a0c88f2ec1a66f421b: Waiting for close lock at 1731972951455Disabling compacts and flushes for region at 1731972951455Disabling writes for close at 1731972951455Writing region close event to WAL at 1731972951471 (+16 ms)Closed at 1731972951471 2024-11-18T23:35:51,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741894_1072 (size=5937) 2024-11-18T23:35:51,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741894_1072 (size=5937) 2024-11-18T23:35:51,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741894_1072 (size=5937) 2024-11-18T23:35:51,522 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/b/6af4ad671e004fd8b51d7c866832b71f to hdfs://localhost:37995/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/b/6af4ad671e004fd8b51d7c866832b71f 2024-11-18T23:35:51,545 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189, size=5.8 K (5937bytes) 2024-11-18T23:35:51,546 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189 2024-11-18T23:35:51,546 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189 after 0ms 2024-11-18T23:35:51,549 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189 took 4ms 2024-11-18T23:35:51,551 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189 so closing down 2024-11-18T23:35:51,551 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:35:51,553 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731972951189.temp 2024-11-18T23:35:51,554 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000003-wal.1731972951189.temp 2024-11-18T23:35:51,554 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:35:51,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741898_1076 (size=5223) 2024-11-18T23:35:51,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741898_1076 (size=5223) 2024-11-18T23:35:51,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741898_1076 (size=5223) 2024-11-18T23:35:51,577 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000003-wal.1731972951189.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-18T23:35:51,578 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000003-wal.1731972951189.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000032 2024-11-18T23:35:51,579 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 29 ms; skipped=2; WAL=hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-18T23:35:51,579 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189, journal: Splitting hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189, size=5.8 K (5937bytes) at 1731972951546Finishing writing output for hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189 so closing down at 1731972951551 (+5 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000003-wal.1731972951189.temp at 1731972951554 (+3 ms)3 split writer threads finished at 1731972951554Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000003-wal.1731972951189.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731972951577 (+23 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000003-wal.1731972951189.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000032 at 1731972951578 (+1 ms)Processed 32 edits across 1 Regions in 29 ms; skipped=2; WAL=hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1731972951579 (+1 ms) 2024-11-18T23:35:51,581 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951189 to hdfs://localhost:37995/hbase/oldWALs/wal.1731972951189 2024-11-18T23:35:51,581 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000032 2024-11-18T23:35:51,582 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:51,584 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:51,607 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951584, exclude list is [], retry=0 2024-11-18T23:35:51,611 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:51,611 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:51,611 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:51,624 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951584 2024-11-18T23:35:51,625 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:35:51,625 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c61bff5d9fab9a0c88f2ec1a66f421b, NAME => 'testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:51,625 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:51,625 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,625 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,628 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,630 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c61bff5d9fab9a0c88f2ec1a66f421b columnFamilyName a 2024-11-18T23:35:51,630 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:51,651 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/a/5bbf1f35060a4ddd98f0dbd1619657ad 2024-11-18T23:35:51,651 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(327): Store=3c61bff5d9fab9a0c88f2ec1a66f421b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:51,652 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,655 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c61bff5d9fab9a0c88f2ec1a66f421b columnFamilyName b 2024-11-18T23:35:51,655 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:51,655 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(327): Store=3c61bff5d9fab9a0c88f2ec1a66f421b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:51,656 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,657 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c61bff5d9fab9a0c88f2ec1a66f421b columnFamilyName c 2024-11-18T23:35:51,657 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:51,688 DEBUG [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/c/50befa934d0048ef840eb66473d6db36 2024-11-18T23:35:51,688 INFO [StoreOpener-3c61bff5d9fab9a0c88f2ec1a66f421b-1 {}] regionserver.HStore(327): Store=3c61bff5d9fab9a0c88f2ec1a66f421b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:51,689 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,696 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,699 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,699 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000032 2024-11-18T23:35:51,710 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000032 2024-11-18T23:35:51,710 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3c61bff5d9fab9a0c88f2ec1a66f421b 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-18T23:35:51,737 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/b/fc3417166c4b4bb09dacb9e0c7547009 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731972951239/Put/seqid=0 2024-11-18T23:35:51,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741900_1078 (size=5958) 2024-11-18T23:35:51,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741900_1078 (size=5958) 2024-11-18T23:35:51,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741900_1078 (size=5958) 2024-11-18T23:35:51,786 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/b/fc3417166c4b4bb09dacb9e0c7547009 2024-11-18T23:35:51,804 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/.tmp/b/fc3417166c4b4bb09dacb9e0c7547009 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/b/fc3417166c4b4bb09dacb9e0c7547009 2024-11-18T23:35:51,811 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/b/fc3417166c4b4bb09dacb9e0c7547009, entries=10, sequenceid=32, filesize=5.8 K 2024-11-18T23:35:51,812 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 3c61bff5d9fab9a0c88f2ec1a66f421b in 101ms, sequenceid=32, compaction requested=false; wal=null 2024-11-18T23:35:51,814 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/0000000000000000032 2024-11-18T23:35:51,815 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,815 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,816 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:51,817 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3c61bff5d9fab9a0c88f2ec1a66f421b 2024-11-18T23:35:51,828 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/3c61bff5d9fab9a0c88f2ec1a66f421b/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-18T23:35:51,831 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3c61bff5d9fab9a0c88f2ec1a66f421b; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64153442, jitterRate=-0.04403921961784363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:51,832 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3c61bff5d9fab9a0c88f2ec1a66f421b: Writing region info on filesystem at 1731972951626Initializing all the Stores at 1731972951626Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972951626Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972951628 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972951628Obtaining lock to block concurrent updates at 1731972951710 (+82 ms)Preparing flush snapshotting stores in 3c61bff5d9fab9a0c88f2ec1a66f421b at 1731972951710Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731972951710Flushing stores of testReplayEditsWrittenViaHRegion,,1731972951083.3c61bff5d9fab9a0c88f2ec1a66f421b. at 1731972951710Flushing 3c61bff5d9fab9a0c88f2ec1a66f421b/b: creating writer at 1731972951710Flushing 3c61bff5d9fab9a0c88f2ec1a66f421b/b: appending metadata at 1731972951736 (+26 ms)Flushing 3c61bff5d9fab9a0c88f2ec1a66f421b/b: closing flushed file at 1731972951736Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5732aaf: reopening flushed file at 1731972951803 (+67 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 3c61bff5d9fab9a0c88f2ec1a66f421b in 101ms, sequenceid=32, compaction requested=false; wal=null at 1731972951812 (+9 ms)Cleaning up temporary data from old regions at 1731972951815 (+3 ms)Region opened successfully at 1731972951832 (+17 ms) 2024-11-18T23:35:51,879 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=417 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:55612 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:37684 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:42916 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:37780 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:43042 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:55768 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1117 (was 1043) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1404 (was 1431), ProcessCount=11 (was 11), AvailableMemoryMB=5225 (was 5306) 2024-11-18T23:35:51,879 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1117 is superior to 1024 2024-11-18T23:35:51,905 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417, OpenFileDescriptor=1117, MaxFileDescriptor=1048576, SystemLoadAverage=1404, ProcessCount=11, AvailableMemoryMB=5218 2024-11-18T23:35:51,905 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1117 is superior to 1024 2024-11-18T23:35:51,937 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:51,942 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:35:51,944 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:35:51,948 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-55675337, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/hregion-55675337, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:51,962 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-55675337/hregion-55675337.1731972951949, exclude list is [], retry=0 2024-11-18T23:35:51,966 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:51,971 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:51,972 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:51,997 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:35:52,002 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-55675337/hregion-55675337.1731972951949 2024-11-18T23:35:52,002 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:35:52,002 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c98ac8e8656dbc0da013d5f27ee419d6, NAME => 'testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/hbase 2024-11-18T23:35:52,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741902_1080 (size=68) 2024-11-18T23:35:52,062 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:52,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741902_1080 (size=68) 2024-11-18T23:35:52,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741902_1080 (size=68) 2024-11-18T23:35:52,067 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,077 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98ac8e8656dbc0da013d5f27ee419d6 columnFamilyName a 2024-11-18T23:35:52,077 DEBUG [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:52,080 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(327): Store=c98ac8e8656dbc0da013d5f27ee419d6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:52,081 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,083 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98ac8e8656dbc0da013d5f27ee419d6 columnFamilyName b 2024-11-18T23:35:52,083 DEBUG [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:52,084 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(327): Store=c98ac8e8656dbc0da013d5f27ee419d6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:52,084 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,088 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98ac8e8656dbc0da013d5f27ee419d6 columnFamilyName c 2024-11-18T23:35:52,088 DEBUG [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:52,089 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(327): Store=c98ac8e8656dbc0da013d5f27ee419d6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:52,090 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,091 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,092 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,093 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,093 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,094 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:52,096 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,102 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:35:52,103 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c98ac8e8656dbc0da013d5f27ee419d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59112053, jitterRate=-0.11916176974773407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:52,104 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c98ac8e8656dbc0da013d5f27ee419d6: Writing region info on filesystem at 1731972952062Initializing all the Stores at 1731972952067 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972952067Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972952067Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972952067Cleaning up temporary data from old regions at 1731972952094 (+27 ms)Region opened successfully at 1731972952104 (+10 ms) 2024-11-18T23:35:52,104 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c98ac8e8656dbc0da013d5f27ee419d6, disabling compactions & flushes 2024-11-18T23:35:52,104 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. 2024-11-18T23:35:52,104 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. 2024-11-18T23:35:52,104 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. after waiting 0 ms 2024-11-18T23:35:52,104 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. 2024-11-18T23:35:52,105 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. 2024-11-18T23:35:52,105 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c98ac8e8656dbc0da013d5f27ee419d6: Waiting for close lock at 1731972952104Disabling compacts and flushes for region at 1731972952104Disabling writes for close at 1731972952104Writing region close event to WAL at 1731972952105 (+1 ms)Closed at 1731972952105 2024-11-18T23:35:52,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741901_1079 (size=93) 2024-11-18T23:35:52,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741901_1079 (size=93) 2024-11-18T23:35:52,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741901_1079 (size=93) 2024-11-18T23:35:52,116 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:35:52,116 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-55675337:(num 1731972951949) 2024-11-18T23:35:52,116 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:35:52,119 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:35:52,140 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120, exclude list is [], retry=0 2024-11-18T23:35:52,148 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:35:52,149 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:35:52,162 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:35:52,167 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120 2024-11-18T23:35:52,167 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:35:52,383 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c98ac8e8656dbc0da013d5f27ee419d6, NAME => 'testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:35:52,392 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,392 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:35:52,392 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,392 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,398 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,401 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98ac8e8656dbc0da013d5f27ee419d6 columnFamilyName a 2024-11-18T23:35:52,402 DEBUG [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:52,406 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(327): Store=c98ac8e8656dbc0da013d5f27ee419d6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:52,406 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,407 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98ac8e8656dbc0da013d5f27ee419d6 columnFamilyName b 2024-11-18T23:35:52,408 DEBUG [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:52,408 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(327): Store=c98ac8e8656dbc0da013d5f27ee419d6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:52,409 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,410 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98ac8e8656dbc0da013d5f27ee419d6 columnFamilyName c 2024-11-18T23:35:52,410 DEBUG [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:35:52,411 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(327): Store=c98ac8e8656dbc0da013d5f27ee419d6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:35:52,411 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,413 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,416 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T23:35:52,421 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,426 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,426 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,427 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:35:52,432 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,434 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c98ac8e8656dbc0da013d5f27ee419d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60393230, jitterRate=-0.10007074475288391}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:35:52,434 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:35:52,435 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c98ac8e8656dbc0da013d5f27ee419d6: Running coprocessor pre-open hook at 1731972952393Writing region info on filesystem at 1731972952393Initializing all the Stores at 1731972952397 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972952397Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972952397Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972952397Cleaning up temporary data from old regions at 1731972952426 (+29 ms)Running coprocessor post-open hooks at 1731972952434 (+8 ms)Region opened successfully at 1731972952435 (+1 ms) 2024-11-18T23:35:52,487 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c98ac8e8656dbc0da013d5f27ee419d6 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-18T23:35:52,488 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c98ac8e8656dbc0da013d5f27ee419d6/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:53,489 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c98ac8e8656dbc0da013d5f27ee419d6/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:53,916 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-18T23:35:53,916 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-18T23:35:53,917 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-18T23:35:53,917 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-18T23:35:53,918 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-18T23:35:53,918 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-18T23:35:54,490 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c98ac8e8656dbc0da013d5f27ee419d6/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:55,491 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c98ac8e8656dbc0da013d5f27ee419d6/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:56,491 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c98ac8e8656dbc0da013d5f27ee419d6/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:57,492 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c98ac8e8656dbc0da013d5f27ee419d6/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:57,895 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:35:58,493 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c98ac8e8656dbc0da013d5f27ee419d6/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:35:59,494 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c98ac8e8656dbc0da013d5f27ee419d6/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:00,495 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c98ac8e8656dbc0da013d5f27ee419d6/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:01,496 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for c98ac8e8656dbc0da013d5f27ee419d6/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:01,498 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c98ac8e8656dbc0da013d5f27ee419d6: 2024-11-18T23:36:01,498 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. 2024-11-18T23:36:01,525 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c98ac8e8656dbc0da013d5f27ee419d6: 2024-11-18T23:36:01,525 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-18T23:36:01,525 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c98ac8e8656dbc0da013d5f27ee419d6, disabling compactions & flushes 2024-11-18T23:36:01,525 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. 2024-11-18T23:36:01,525 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. 2024-11-18T23:36:01,525 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. after waiting 0 ms 2024-11-18T23:36:01,525 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. 2024-11-18T23:36:01,526 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. 2024-11-18T23:36:01,526 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. 2024-11-18T23:36:01,526 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c98ac8e8656dbc0da013d5f27ee419d6: Waiting for close lock at 1731972961525Running coprocessor pre-close hooks at 1731972961525Disabling compacts and flushes for region at 1731972961525Disabling writes for close at 1731972961525Writing region close event to WAL at 1731972961526 (+1 ms)Running coprocessor post-close hooks at 1731972961526Closed at 1731972961526 2024-11-18T23:36:01,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741903_1081 (size=3941) 2024-11-18T23:36:01,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741903_1081 (size=3941) 2024-11-18T23:36:01,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741903_1081 (size=3941) 2024-11-18T23:36:01,569 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120, size=3.8 K (3941bytes) 2024-11-18T23:36:01,570 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120 2024-11-18T23:36:01,570 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120 after 0ms 2024-11-18T23:36:01,588 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120 took 19ms 2024-11-18T23:36:01,622 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120 so closing down 2024-11-18T23:36:01,622 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:36:01,629 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731972952120.temp 2024-11-18T23:36:01,634 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000004-wal.1731972952120.temp 2024-11-18T23:36:01,634 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:36:01,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741904_1082 (size=2983) 2024-11-18T23:36:01,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741904_1082 (size=2983) 2024-11-18T23:36:01,713 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000004-wal.1731972952120.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-18T23:36:01,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741904_1082 (size=2983) 2024-11-18T23:36:01,724 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000004-wal.1731972952120.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000026 2024-11-18T23:36:01,725 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 125 ms; skipped=3; WAL=hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-18T23:36:01,725 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120, journal: Splitting hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120, size=3.8 K (3941bytes) at 1731972961570Finishing writing output for hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120 so closing down at 1731972961622 (+52 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000004-wal.1731972952120.temp at 1731972961634 (+12 ms)3 split writer threads finished at 1731972961635 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000004-wal.1731972952120.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731972961713 (+78 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000004-wal.1731972952120.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000026 at 1731972961724 (+11 ms)Processed 23 edits across 1 Regions in 125 ms; skipped=3; WAL=hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1731972961725 (+1 ms) 2024-11-18T23:36:01,745 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972952120 to hdfs://localhost:37995/hbase/oldWALs/wal.1731972952120 2024-11-18T23:36:01,752 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000026 2024-11-18T23:36:01,752 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:36:01,755 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:01,777 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972961756, exclude list is [], retry=0 2024-11-18T23:36:01,785 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:01,786 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:01,786 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:01,832 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972961756 2024-11-18T23:36:01,836 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:42157:42157)] 2024-11-18T23:36:01,837 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c98ac8e8656dbc0da013d5f27ee419d6, NAME => 'testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:36:01,838 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:01,838 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:01,840 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:01,840 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:01,875 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:01,884 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98ac8e8656dbc0da013d5f27ee419d6 columnFamilyName a 2024-11-18T23:36:01,885 DEBUG [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:01,890 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(327): Store=c98ac8e8656dbc0da013d5f27ee419d6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:01,890 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:01,899 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98ac8e8656dbc0da013d5f27ee419d6 columnFamilyName b 2024-11-18T23:36:01,899 DEBUG [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:01,905 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(327): Store=c98ac8e8656dbc0da013d5f27ee419d6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:01,905 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:01,913 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98ac8e8656dbc0da013d5f27ee419d6 columnFamilyName c 2024-11-18T23:36:01,913 DEBUG [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:01,920 INFO [StoreOpener-c98ac8e8656dbc0da013d5f27ee419d6-1 {}] regionserver.HStore(327): Store=c98ac8e8656dbc0da013d5f27ee419d6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:01,924 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:01,932 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:01,960 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:01,972 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000026 2024-11-18T23:36:01,989 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000026 2024-11-18T23:36:01,990 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c98ac8e8656dbc0da013d5f27ee419d6 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-18T23:36:02,045 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/.tmp/a/ae2b4bdba2ce4f74b479d987c4555e4f is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731972961505/Put/seqid=0 2024-11-18T23:36:02,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741906_1084 (size=5523) 2024-11-18T23:36:02,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741906_1084 (size=5523) 2024-11-18T23:36:02,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741906_1084 (size=5523) 2024-11-18T23:36:02,145 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/.tmp/a/ae2b4bdba2ce4f74b479d987c4555e4f 2024-11-18T23:36:02,197 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/.tmp/b/1ef33fe085824ce9855278446468b12e is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731972961498/Put/seqid=0 2024-11-18T23:36:02,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741907_1085 (size=5524) 2024-11-18T23:36:02,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741907_1085 (size=5524) 2024-11-18T23:36:02,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741907_1085 (size=5524) 2024-11-18T23:36:02,306 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/.tmp/b/1ef33fe085824ce9855278446468b12e 2024-11-18T23:36:02,370 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/.tmp/c/37e2e973eae94c3da4843c7efe2429db is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731972961502/Put/seqid=0 2024-11-18T23:36:02,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741908_1086 (size=5457) 2024-11-18T23:36:02,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741908_1086 (size=5457) 2024-11-18T23:36:02,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741908_1086 (size=5457) 2024-11-18T23:36:02,472 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/.tmp/c/37e2e973eae94c3da4843c7efe2429db 2024-11-18T23:36:02,500 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/.tmp/a/ae2b4bdba2ce4f74b479d987c4555e4f as hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/a/ae2b4bdba2ce4f74b479d987c4555e4f 2024-11-18T23:36:02,511 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/a/ae2b4bdba2ce4f74b479d987c4555e4f, entries=7, sequenceid=26, filesize=5.4 K 2024-11-18T23:36:02,512 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/.tmp/b/1ef33fe085824ce9855278446468b12e as hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/b/1ef33fe085824ce9855278446468b12e 2024-11-18T23:36:02,519 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/b/1ef33fe085824ce9855278446468b12e, entries=7, sequenceid=26, filesize=5.4 K 2024-11-18T23:36:02,520 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/.tmp/c/37e2e973eae94c3da4843c7efe2429db as hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/c/37e2e973eae94c3da4843c7efe2429db 2024-11-18T23:36:02,531 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/c/37e2e973eae94c3da4843c7efe2429db, entries=6, sequenceid=26, filesize=5.3 K 2024-11-18T23:36:02,532 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for c98ac8e8656dbc0da013d5f27ee419d6 in 542ms, sequenceid=26, compaction requested=false; wal=null 2024-11-18T23:36:02,533 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/0000000000000000026 2024-11-18T23:36:02,534 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:02,535 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:02,536 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:36:02,538 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:02,548 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsAfterAbortingFlush/c98ac8e8656dbc0da013d5f27ee419d6/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-18T23:36:02,550 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c98ac8e8656dbc0da013d5f27ee419d6; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64322577, jitterRate=-0.041518911719322205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:36:02,550 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c98ac8e8656dbc0da013d5f27ee419d6 2024-11-18T23:36:02,552 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c98ac8e8656dbc0da013d5f27ee419d6: Running coprocessor pre-open hook at 1731972961840Writing region info on filesystem at 1731972961840Initializing all the Stores at 1731972961858 (+18 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972961858Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972961874 (+16 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972961874Obtaining lock to block concurrent updates at 1731972961990 (+116 ms)Preparing flush snapshotting stores in c98ac8e8656dbc0da013d5f27ee419d6 at 1731972961990Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731972961990Flushing stores of testReplayEditsAfterAbortingFlush,,1731972951938.c98ac8e8656dbc0da013d5f27ee419d6. at 1731972961990Flushing c98ac8e8656dbc0da013d5f27ee419d6/a: creating writer at 1731972961991 (+1 ms)Flushing c98ac8e8656dbc0da013d5f27ee419d6/a: appending metadata at 1731972962045 (+54 ms)Flushing c98ac8e8656dbc0da013d5f27ee419d6/a: closing flushed file at 1731972962045Flushing c98ac8e8656dbc0da013d5f27ee419d6/b: creating writer at 1731972962172 (+127 ms)Flushing c98ac8e8656dbc0da013d5f27ee419d6/b: appending metadata at 1731972962196 (+24 ms)Flushing c98ac8e8656dbc0da013d5f27ee419d6/b: closing flushed file at 1731972962196Flushing c98ac8e8656dbc0da013d5f27ee419d6/c: creating writer at 1731972962331 (+135 ms)Flushing c98ac8e8656dbc0da013d5f27ee419d6/c: appending metadata at 1731972962369 (+38 ms)Flushing c98ac8e8656dbc0da013d5f27ee419d6/c: closing flushed file at 1731972962369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e4dd65b: reopening flushed file at 1731972962490 (+121 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75481c39: reopening flushed file at 1731972962511 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63735638: reopening flushed file at 1731972962519 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for c98ac8e8656dbc0da013d5f27ee419d6 in 542ms, sequenceid=26, compaction requested=false; wal=null at 1731972962532 (+13 ms)Cleaning up temporary data from old regions at 1731972962535 (+3 ms)Running coprocessor post-open hooks at 1731972962550 (+15 ms)Region opened successfully at 1731972962552 (+2 ms) 2024-11-18T23:36:02,583 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420 (was 417) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:59962 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:37724 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:50404 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:36596 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:36610 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:45280 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1186 (was 1117) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1321 (was 1404), ProcessCount=11 (was 11), AvailableMemoryMB=6187 (was 5218) - AvailableMemoryMB LEAK? - 2024-11-18T23:36:02,583 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-18T23:36:02,600 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=420, OpenFileDescriptor=1186, MaxFileDescriptor=1048576, SystemLoadAverage=1321, ProcessCount=11, AvailableMemoryMB=6184 2024-11-18T23:36:02,600 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-18T23:36:02,620 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:36:02,623 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:36:02,624 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:36:02,627 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-21789623, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/hregion-21789623, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:02,647 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-21789623/hregion-21789623.1731972962628, exclude list is [], retry=0 2024-11-18T23:36:02,651 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:02,656 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:02,670 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:02,686 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-21789623/hregion-21789623.1731972962628 2024-11-18T23:36:02,686 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:36:02,686 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bcdba99c7c895df6c3cabcb16cfbd25d, NAME => 'testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/hbase 2024-11-18T23:36:02,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741910_1088 (size=61) 2024-11-18T23:36:02,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741910_1088 (size=61) 2024-11-18T23:36:02,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741910_1088 (size=61) 2024-11-18T23:36:03,129 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:03,131 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,133 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcdba99c7c895df6c3cabcb16cfbd25d columnFamilyName a 2024-11-18T23:36:03,133 DEBUG [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:03,133 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] regionserver.HStore(327): Store=bcdba99c7c895df6c3cabcb16cfbd25d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:03,134 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,135 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,135 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,136 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,136 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,138 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,140 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:36:03,140 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bcdba99c7c895df6c3cabcb16cfbd25d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66214609, jitterRate=-0.013325437903404236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:36:03,141 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bcdba99c7c895df6c3cabcb16cfbd25d: Writing region info on filesystem at 1731972963130Initializing all the Stores at 1731972963131 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972963131Cleaning up temporary data from old regions at 1731972963136 (+5 ms)Region opened successfully at 1731972963141 (+5 ms) 2024-11-18T23:36:03,141 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bcdba99c7c895df6c3cabcb16cfbd25d, disabling compactions & flushes 2024-11-18T23:36:03,141 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. 2024-11-18T23:36:03,141 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. 2024-11-18T23:36:03,141 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. after waiting 0 ms 2024-11-18T23:36:03,142 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. 2024-11-18T23:36:03,142 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. 2024-11-18T23:36:03,142 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bcdba99c7c895df6c3cabcb16cfbd25d: Waiting for close lock at 1731972963141Disabling compacts and flushes for region at 1731972963141Disabling writes for close at 1731972963142 (+1 ms)Writing region close event to WAL at 1731972963142Closed at 1731972963142 2024-11-18T23:36:03,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741909_1087 (size=93) 2024-11-18T23:36:03,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741909_1087 (size=93) 2024-11-18T23:36:03,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741909_1087 (size=93) 2024-11-18T23:36:03,150 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:36:03,150 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-21789623:(num 1731972962628) 2024-11-18T23:36:03,150 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:36:03,152 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:03,172 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153, exclude list is [], retry=0 2024-11-18T23:36:03,175 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:03,175 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:03,175 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:03,189 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153 2024-11-18T23:36:03,189 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:36:03,189 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bcdba99c7c895df6c3cabcb16cfbd25d, NAME => 'testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:36:03,190 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:03,190 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,190 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,193 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,194 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcdba99c7c895df6c3cabcb16cfbd25d columnFamilyName a 2024-11-18T23:36:03,194 DEBUG [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:03,195 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] regionserver.HStore(327): Store=bcdba99c7c895df6c3cabcb16cfbd25d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:03,195 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,196 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,197 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,198 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,198 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,200 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,202 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bcdba99c7c895df6c3cabcb16cfbd25d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66402461, jitterRate=-0.010526224970817566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:36:03,203 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bcdba99c7c895df6c3cabcb16cfbd25d: Writing region info on filesystem at 1731972963190Initializing all the Stores at 1731972963192 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972963192Cleaning up temporary data from old regions at 1731972963198 (+6 ms)Region opened successfully at 1731972963203 (+5 ms) 2024-11-18T23:36:03,222 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bcdba99c7c895df6c3cabcb16cfbd25d, disabling compactions & flushes 2024-11-18T23:36:03,222 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. 2024-11-18T23:36:03,222 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. 2024-11-18T23:36:03,222 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. after waiting 0 ms 2024-11-18T23:36:03,222 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. 2024-11-18T23:36:03,223 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. 2024-11-18T23:36:03,223 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. 2024-11-18T23:36:03,223 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bcdba99c7c895df6c3cabcb16cfbd25d: Waiting for close lock at 1731972963222Disabling compacts and flushes for region at 1731972963222Disabling writes for close at 1731972963222Writing region close event to WAL at 1731972963222Closed at 1731972963223 (+1 ms) 2024-11-18T23:36:03,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741911_1089 (size=1623) 2024-11-18T23:36:03,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741911_1089 (size=1623) 2024-11-18T23:36:03,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741911_1089 (size=1623) 2024-11-18T23:36:03,249 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153, size=1.6 K (1623bytes) 2024-11-18T23:36:03,249 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153 2024-11-18T23:36:03,250 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153 after 0ms 2024-11-18T23:36:03,253 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153 took 4ms 2024-11-18T23:36:03,255 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153 so closing down 2024-11-18T23:36:03,255 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:36:03,256 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731972963153.temp 2024-11-18T23:36:03,258 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000003-wal.1731972963153.temp 2024-11-18T23:36:03,258 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:36:03,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741912_1090 (size=1623) 2024-11-18T23:36:03,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741912_1090 (size=1623) 2024-11-18T23:36:03,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741912_1090 (size=1623) 2024-11-18T23:36:03,266 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000003-wal.1731972963153.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-18T23:36:03,267 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000003-wal.1731972963153.temp to hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000012 2024-11-18T23:36:03,268 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-18T23:36:03,268 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153, journal: Splitting hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153, size=1.6 K (1623bytes) at 1731972963249Finishing writing output for hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153 so closing down at 1731972963255 (+6 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000003-wal.1731972963153.temp at 1731972963258 (+3 ms)3 split writer threads finished at 1731972963258Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000003-wal.1731972963153.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731972963266 (+8 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000003-wal.1731972963153.temp to hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000012 at 1731972963268 (+2 ms)Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1731972963268 2024-11-18T23:36:03,270 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963153 to hdfs://localhost:37995/hbase/oldWALs/wal.1731972963153 2024-11-18T23:36:03,271 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000012 2024-11-18T23:36:03,907 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:36:03,913 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:03,932 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963913, exclude list is [], retry=0 2024-11-18T23:36:03,941 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:03,942 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:03,942 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:03,964 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963913 2024-11-18T23:36:03,965 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:36:03,965 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bcdba99c7c895df6c3cabcb16cfbd25d, NAME => 'testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:36:03,965 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:03,965 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,965 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,980 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,986 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcdba99c7c895df6c3cabcb16cfbd25d columnFamilyName a 2024-11-18T23:36:03,987 DEBUG [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:03,989 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] regionserver.HStore(327): Store=bcdba99c7c895df6c3cabcb16cfbd25d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:03,989 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:03,990 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,006 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,016 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000012 2024-11-18T23:36:04,020 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000012 2024-11-18T23:36:04,020 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bcdba99c7c895df6c3cabcb16cfbd25d 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-18T23:36:04,051 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/.tmp/a/45eb5f2494be46a59d15ffe6ea4efd1a is 79, key is testDatalossWhenInputError/a:x0/1731972963203/Put/seqid=0 2024-11-18T23:36:04,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741914_1092 (size=5808) 2024-11-18T23:36:04,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741914_1092 (size=5808) 2024-11-18T23:36:04,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741914_1092 (size=5808) 2024-11-18T23:36:04,069 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/.tmp/a/45eb5f2494be46a59d15ffe6ea4efd1a 2024-11-18T23:36:04,080 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/.tmp/a/45eb5f2494be46a59d15ffe6ea4efd1a as hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/a/45eb5f2494be46a59d15ffe6ea4efd1a 2024-11-18T23:36:04,094 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/a/45eb5f2494be46a59d15ffe6ea4efd1a, entries=10, sequenceid=12, filesize=5.7 K 2024-11-18T23:36:04,094 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for bcdba99c7c895df6c3cabcb16cfbd25d in 74ms, sequenceid=12, compaction requested=false; wal=null 2024-11-18T23:36:04,095 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/0000000000000000012 2024-11-18T23:36:04,096 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,097 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,100 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,103 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-18T23:36:04,105 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bcdba99c7c895df6c3cabcb16cfbd25d; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58992946, jitterRate=-0.12093660235404968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:36:04,105 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bcdba99c7c895df6c3cabcb16cfbd25d: Writing region info on filesystem at 1731972963965Initializing all the Stores at 1731972963974 (+9 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972963974Obtaining lock to block concurrent updates at 1731972964020 (+46 ms)Preparing flush snapshotting stores in bcdba99c7c895df6c3cabcb16cfbd25d at 1731972964020Finished memstore snapshotting testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731972964020Flushing stores of testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d. at 1731972964020Flushing bcdba99c7c895df6c3cabcb16cfbd25d/a: creating writer at 1731972964020Flushing bcdba99c7c895df6c3cabcb16cfbd25d/a: appending metadata at 1731972964050 (+30 ms)Flushing bcdba99c7c895df6c3cabcb16cfbd25d/a: closing flushed file at 1731972964050Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23739dbc: reopening flushed file at 1731972964078 (+28 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for bcdba99c7c895df6c3cabcb16cfbd25d in 74ms, sequenceid=12, compaction requested=false; wal=null at 1731972964094 (+16 ms)Cleaning up temporary data from old regions at 1731972964097 (+3 ms)Region opened successfully at 1731972964105 (+8 ms) 2024-11-18T23:36:04,108 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bcdba99c7c895df6c3cabcb16cfbd25d, NAME => 'testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:36:04,109 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731972962621.bcdba99c7c895df6c3cabcb16cfbd25d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:04,109 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,109 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,114 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,115 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcdba99c7c895df6c3cabcb16cfbd25d columnFamilyName a 2024-11-18T23:36:04,115 DEBUG [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:04,130 DEBUG [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/a/45eb5f2494be46a59d15ffe6ea4efd1a 2024-11-18T23:36:04,131 INFO [StoreOpener-bcdba99c7c895df6c3cabcb16cfbd25d-1 {}] regionserver.HStore(327): Store=bcdba99c7c895df6c3cabcb16cfbd25d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:04,131 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,132 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,133 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,133 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,133 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,136 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bcdba99c7c895df6c3cabcb16cfbd25d 2024-11-18T23:36:04,151 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testDatalossWhenInputError/bcdba99c7c895df6c3cabcb16cfbd25d/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-18T23:36:04,152 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bcdba99c7c895df6c3cabcb16cfbd25d; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73578325, jitterRate=0.09640248119831085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T23:36:04,152 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bcdba99c7c895df6c3cabcb16cfbd25d: Writing region info on filesystem at 1731972964109Initializing all the Stores at 1731972964113 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972964113Cleaning up temporary data from old regions at 1731972964133 (+20 ms)Region opened successfully at 1731972964152 (+19 ms) 2024-11-18T23:36:04,178 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=430 (was 420) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:50474 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:36144 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:37806 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:50404 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:36596 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-330010681-172.17.0.2-1731972917172:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1186) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1321 (was 1321), ProcessCount=11 (was 11), AvailableMemoryMB=5880 (was 6184) 2024-11-18T23:36:04,179 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-18T23:36:04,198 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=430, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=1321, ProcessCount=11, AvailableMemoryMB=5876 2024-11-18T23:36:04,198 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-18T23:36:04,231 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:36:04,237 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:36:04,238 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:36:04,288 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23039856, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/hregion-23039856, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:04,312 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23039856/hregion-23039856.1731972964289, exclude list is [], retry=0 2024-11-18T23:36:04,325 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:04,334 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:04,336 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:04,364 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23039856/hregion-23039856.1731972964289 2024-11-18T23:36:04,369 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157)] 2024-11-18T23:36:04,369 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 739f647798859799f2c1a3c7515c242f, NAME => 'testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/hbase 2024-11-18T23:36:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741916_1094 (size=63) 2024-11-18T23:36:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741916_1094 (size=63) 2024-11-18T23:36:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741916_1094 (size=63) 2024-11-18T23:36:04,820 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:04,822 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,826 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739f647798859799f2c1a3c7515c242f columnFamilyName a 2024-11-18T23:36:04,826 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:04,826 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(327): Store=739f647798859799f2c1a3c7515c242f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:04,827 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,828 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739f647798859799f2c1a3c7515c242f columnFamilyName b 2024-11-18T23:36:04,828 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:04,828 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(327): Store=739f647798859799f2c1a3c7515c242f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:04,828 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,830 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739f647798859799f2c1a3c7515c242f columnFamilyName c 2024-11-18T23:36:04,830 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:04,831 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(327): Store=739f647798859799f2c1a3c7515c242f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:04,831 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,832 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,832 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,834 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,834 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,834 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:36:04,835 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,838 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:36:04,839 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 739f647798859799f2c1a3c7515c242f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59464745, jitterRate=-0.11390624940395355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:36:04,839 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 739f647798859799f2c1a3c7515c242f: Writing region info on filesystem at 1731972964820Initializing all the Stores at 1731972964821 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972964821Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972964822 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972964822Cleaning up temporary data from old regions at 1731972964834 (+12 ms)Region opened successfully at 1731972964839 (+5 ms) 2024-11-18T23:36:04,840 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 739f647798859799f2c1a3c7515c242f, disabling compactions & flushes 2024-11-18T23:36:04,840 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. 2024-11-18T23:36:04,840 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. 2024-11-18T23:36:04,840 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. after waiting 0 ms 2024-11-18T23:36:04,840 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. 2024-11-18T23:36:04,840 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. 2024-11-18T23:36:04,840 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 739f647798859799f2c1a3c7515c242f: Waiting for close lock at 1731972964839Disabling compacts and flushes for region at 1731972964839Disabling writes for close at 1731972964840 (+1 ms)Writing region close event to WAL at 1731972964840Closed at 1731972964840 2024-11-18T23:36:04,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741915_1093 (size=93) 2024-11-18T23:36:04,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741915_1093 (size=93) 2024-11-18T23:36:04,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741915_1093 (size=93) 2024-11-18T23:36:04,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:36:04,848 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23039856:(num 1731972964289) 2024-11-18T23:36:04,849 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:36:04,852 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:04,867 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852, exclude list is [], retry=0 2024-11-18T23:36:04,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:04,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:04,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:04,889 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 2024-11-18T23:36:04,892 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723)] 2024-11-18T23:36:04,893 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 739f647798859799f2c1a3c7515c242f, NAME => 'testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:36:04,893 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:04,893 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,893 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,900 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,902 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739f647798859799f2c1a3c7515c242f columnFamilyName a 2024-11-18T23:36:04,903 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:04,903 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(327): Store=739f647798859799f2c1a3c7515c242f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:04,904 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,907 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739f647798859799f2c1a3c7515c242f columnFamilyName b 2024-11-18T23:36:04,907 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:04,919 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(327): Store=739f647798859799f2c1a3c7515c242f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:04,920 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,921 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739f647798859799f2c1a3c7515c242f columnFamilyName c 2024-11-18T23:36:04,921 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:04,922 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(327): Store=739f647798859799f2c1a3c7515c242f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:04,922 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,923 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,924 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,930 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,930 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,930 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:36:04,932 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:04,933 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 739f647798859799f2c1a3c7515c242f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71511790, jitterRate=0.06560871005058289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:36:04,933 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 739f647798859799f2c1a3c7515c242f: Writing region info on filesystem at 1731972964893Initializing all the Stores at 1731972964895 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972964895Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972964896 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972964896Cleaning up temporary data from old regions at 1731972964930 (+34 ms)Region opened successfully at 1731972964933 (+3 ms) 2024-11-18T23:36:04,938 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731972964938/Put/seqid=0 2024-11-18T23:36:04,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741918_1096 (size=4875) 2024-11-18T23:36:04,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741918_1096 (size=4875) 2024-11-18T23:36:04,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741918_1096 (size=4875) 2024-11-18T23:36:05,367 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731972965367/Put/seqid=0 2024-11-18T23:36:05,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741919_1097 (size=4875) 2024-11-18T23:36:05,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741919_1097 (size=4875) 2024-11-18T23:36:05,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741919_1097 (size=4875) 2024-11-18T23:36:05,388 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731972965388/Put/seqid=0 2024-11-18T23:36:05,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741920_1098 (size=4875) 2024-11-18T23:36:05,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741920_1098 (size=4875) 2024-11-18T23:36:05,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741920_1098 (size=4875) 2024-11-18T23:36:05,402 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 739f647798859799f2c1a3c7515c242f/a 2024-11-18T23:36:05,406 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-18T23:36:05,406 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-18T23:36:05,406 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 739f647798859799f2c1a3c7515c242f/a 2024-11-18T23:36:05,411 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-18T23:36:05,411 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-18T23:36:05,411 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 739f647798859799f2c1a3c7515c242f/a 2024-11-18T23:36:05,415 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-18T23:36:05,415 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-18T23:36:05,415 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 739f647798859799f2c1a3c7515c242f 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-18T23:36:05,433 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/.tmp/a/726083c7a23146958e000e610e572099 is 55, key is testCompactedBulkLoadedFiles/a:a/1731972964934/Put/seqid=0 2024-11-18T23:36:05,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741921_1099 (size=5107) 2024-11-18T23:36:05,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741921_1099 (size=5107) 2024-11-18T23:36:05,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741921_1099 (size=5107) 2024-11-18T23:36:05,440 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/.tmp/a/726083c7a23146958e000e610e572099 2024-11-18T23:36:05,447 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/.tmp/a/726083c7a23146958e000e610e572099 as hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/726083c7a23146958e000e610e572099 2024-11-18T23:36:05,453 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/726083c7a23146958e000e610e572099, entries=1, sequenceid=4, filesize=5.0 K 2024-11-18T23:36:05,454 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 739f647798859799f2c1a3c7515c242f in 39ms, sequenceid=4, compaction requested=false 2024-11-18T23:36:05,454 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 739f647798859799f2c1a3c7515c242f: 2024-11-18T23:36:05,456 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/e8201779ddf34182ae213e7e493a34c1_SeqId_4_ 2024-11-18T23:36:05,457 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_ 2024-11-18T23:36:05,458 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/fccc75fd35fa4a3e929246e61857cd50_SeqId_4_ 2024-11-18T23:36:05,458 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile0 into 739f647798859799f2c1a3c7515c242f/a as hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/e8201779ddf34182ae213e7e493a34c1_SeqId_4_ - updating store file list. 2024-11-18T23:36:05,463 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e8201779ddf34182ae213e7e493a34c1_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-18T23:36:05,463 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/e8201779ddf34182ae213e7e493a34c1_SeqId_4_ into 739f647798859799f2c1a3c7515c242f/a 2024-11-18T23:36:05,463 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile0 into 739f647798859799f2c1a3c7515c242f/a (new location: hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/e8201779ddf34182ae213e7e493a34c1_SeqId_4_) 2024-11-18T23:36:05,464 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile1 into 739f647798859799f2c1a3c7515c242f/a as hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_ - updating store file list. 2024-11-18T23:36:05,468 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-18T23:36:05,468 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_ into 739f647798859799f2c1a3c7515c242f/a 2024-11-18T23:36:05,468 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile1 into 739f647798859799f2c1a3c7515c242f/a (new location: hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_) 2024-11-18T23:36:05,469 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile2 into 739f647798859799f2c1a3c7515c242f/a as hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/fccc75fd35fa4a3e929246e61857cd50_SeqId_4_ - updating store file list. 2024-11-18T23:36:05,472 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for fccc75fd35fa4a3e929246e61857cd50_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-18T23:36:05,472 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/fccc75fd35fa4a3e929246e61857cd50_SeqId_4_ into 739f647798859799f2c1a3c7515c242f/a 2024-11-18T23:36:05,472 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37995/hbase/testCompactedBulkLoadedFiles/hfile2 into 739f647798859799f2c1a3c7515c242f/a (new location: hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/fccc75fd35fa4a3e929246e61857cd50_SeqId_4_) 2024-11-18T23:36:05,479 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:36:05,479 DEBUG [Time-limited test {}] regionserver.HStore(1541): 739f647798859799f2c1a3c7515c242f/a is initiating major compaction (all files) 2024-11-18T23:36:05,479 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 739f647798859799f2c1a3c7515c242f/a in testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. 2024-11-18T23:36:05,479 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/726083c7a23146958e000e610e572099, hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/e8201779ddf34182ae213e7e493a34c1_SeqId_4_, hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_, hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/fccc75fd35fa4a3e929246e61857cd50_SeqId_4_] into tmpdir=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/.tmp, totalSize=19.3 K 2024-11-18T23:36:05,480 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 726083c7a23146958e000e610e572099, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731972964934 2024-11-18T23:36:05,480 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e8201779ddf34182ae213e7e493a34c1_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-18T23:36:05,481 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-18T23:36:05,481 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fccc75fd35fa4a3e929246e61857cd50_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-18T23:36:05,496 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/.tmp/a/8aae4544c338479aa1ca6008891e9591 is 55, key is testCompactedBulkLoadedFiles/a:a/1731972964934/Put/seqid=0 2024-11-18T23:36:05,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741922_1100 (size=6154) 2024-11-18T23:36:05,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741922_1100 (size=6154) 2024-11-18T23:36:05,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741922_1100 (size=6154) 2024-11-18T23:36:05,509 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/.tmp/a/8aae4544c338479aa1ca6008891e9591 as hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/8aae4544c338479aa1ca6008891e9591 2024-11-18T23:36:05,517 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 739f647798859799f2c1a3c7515c242f/a of 739f647798859799f2c1a3c7515c242f into 8aae4544c338479aa1ca6008891e9591(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:36:05,517 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 739f647798859799f2c1a3c7515c242f: 2024-11-18T23:36:05,517 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-18T23:36:05,517 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-18T23:36:05,563 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852, size=0 (0bytes) 2024-11-18T23:36:05,563 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 might be still open, length is 0 2024-11-18T23:36:05,563 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 2024-11-18T23:36:05,564 WARN [IPC Server handler 3 on default port 37995 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-18T23:36:05,564 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 after 1ms 2024-11-18T23:36:07,029 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:36:07,406 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:36188 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:35345:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36188 dst: /127.0.0.1:35345 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35345 remote=/127.0.0.1:36188]. Total timeout mills is 60000, 58110 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:07,407 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:50506 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50506 dst: /127.0.0.1:36973 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:07,407 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:37846 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:39707:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37846 dst: /127.0.0.1:39707 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:07,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741917_1101 (size=1659) 2024-11-18T23:36:07,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741917_1101 (size=1659) 2024-11-18T23:36:07,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741917_1101 (size=1659) 2024-11-18T23:36:09,173 INFO [master/9d96596e55a9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T23:36:09,173 INFO [master/9d96596e55a9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T23:36:09,565 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 after 4002ms 2024-11-18T23:36:09,569 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 took 4006ms 2024-11-18T23:36:09,577 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852; continuing. 2024-11-18T23:36:09,577 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 so closing down 2024-11-18T23:36:09,577 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:36:09,586 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731972964852.temp 2024-11-18T23:36:09,596 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000003-wal.1731972964852.temp 2024-11-18T23:36:09,596 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:36:09,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741923_1102 (size=698) 2024-11-18T23:36:09,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741923_1102 (size=698) 2024-11-18T23:36:09,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741923_1102 (size=698) 2024-11-18T23:36:09,632 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000003-wal.1731972964852.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-18T23:36:09,648 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000003-wal.1731972964852.temp to hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000008 2024-11-18T23:36:09,649 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 75 ms; skipped=3; WAL=hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852, size=0, length=0, corrupted=false, cancelled=false 2024-11-18T23:36:09,649 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852, journal: Splitting hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852, size=0 (0bytes) at 1731972965563Finishing writing output for hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 so closing down at 1731972969577 (+4014 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000003-wal.1731972964852.temp at 1731972969596 (+19 ms)3 split writer threads finished at 1731972969596Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000003-wal.1731972964852.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731972969632 (+36 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000003-wal.1731972964852.temp to hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000008 at 1731972969649 (+17 ms)Processed 5 edits across 1 Regions in 75 ms; skipped=3; WAL=hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852, size=0, length=0, corrupted=false, cancelled=false at 1731972969649 2024-11-18T23:36:09,651 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 to hdfs://localhost:37995/hbase/oldWALs/wal.1731972964852 2024-11-18T23:36:09,653 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000008 2024-11-18T23:36:09,653 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:36:09,658 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:09,678 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972969659, exclude list is [], retry=0 2024-11-18T23:36:09,685 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:09,686 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:09,686 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:09,716 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972969659 2024-11-18T23:36:09,720 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42361:42361),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157)] 2024-11-18T23:36:09,721 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 739f647798859799f2c1a3c7515c242f, NAME => 'testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:36:09,721 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:09,721 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,721 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,734 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,735 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739f647798859799f2c1a3c7515c242f columnFamilyName a 2024-11-18T23:36:09,735 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:09,805 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/726083c7a23146958e000e610e572099 2024-11-18T23:36:09,839 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/8aae4544c338479aa1ca6008891e9591 2024-11-18T23:36:09,853 DEBUG [StoreFileOpener-739f647798859799f2c1a3c7515c242f-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e8201779ddf34182ae213e7e493a34c1_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-18T23:36:09,854 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/e8201779ddf34182ae213e7e493a34c1_SeqId_4_ 2024-11-18T23:36:09,861 DEBUG [StoreFileOpener-739f647798859799f2c1a3c7515c242f-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-18T23:36:09,862 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_ 2024-11-18T23:36:09,895 DEBUG [StoreFileOpener-739f647798859799f2c1a3c7515c242f-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for fccc75fd35fa4a3e929246e61857cd50_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-18T23:36:09,895 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/fccc75fd35fa4a3e929246e61857cd50_SeqId_4_ 2024-11-18T23:36:09,895 WARN [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/726083c7a23146958e000e610e572099 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@556ed36a 2024-11-18T23:36:09,895 WARN [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/e8201779ddf34182ae213e7e493a34c1_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@556ed36a 2024-11-18T23:36:09,895 WARN [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@556ed36a 2024-11-18T23:36:09,895 WARN [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/fccc75fd35fa4a3e929246e61857cd50_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@556ed36a 2024-11-18T23:36:09,896 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/726083c7a23146958e000e610e572099, hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/e8201779ddf34182ae213e7e493a34c1_SeqId_4_, hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_, hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/fccc75fd35fa4a3e929246e61857cd50_SeqId_4_] to archive 2024-11-18T23:36:09,901 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:36:09,904 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/726083c7a23146958e000e610e572099 to hdfs://localhost:37995/hbase/archive/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/726083c7a23146958e000e610e572099 2024-11-18T23:36:09,908 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/e8201779ddf34182ae213e7e493a34c1_SeqId_4_ to hdfs://localhost:37995/hbase/archive/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/e8201779ddf34182ae213e7e493a34c1_SeqId_4_ 2024-11-18T23:36:09,910 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_ to hdfs://localhost:37995/hbase/archive/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_ 2024-11-18T23:36:09,913 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/fccc75fd35fa4a3e929246e61857cd50_SeqId_4_ to hdfs://localhost:37995/hbase/archive/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/a/fccc75fd35fa4a3e929246e61857cd50_SeqId_4_ 2024-11-18T23:36:09,913 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(327): Store=739f647798859799f2c1a3c7515c242f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:09,913 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,915 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739f647798859799f2c1a3c7515c242f columnFamilyName b 2024-11-18T23:36:09,916 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:09,916 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(327): Store=739f647798859799f2c1a3c7515c242f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:09,917 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,918 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739f647798859799f2c1a3c7515c242f columnFamilyName c 2024-11-18T23:36:09,919 DEBUG [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:09,920 INFO [StoreOpener-739f647798859799f2c1a3c7515c242f-1 {}] regionserver.HStore(327): Store=739f647798859799f2c1a3c7515c242f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:09,920 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,924 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,942 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,949 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000008 2024-11-18T23:36:09,955 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 739f647798859799f2c1a3c7515c242f : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "739f647798859799f2c1a3c7515c242f" family_name: "a" compaction_input: "726083c7a23146958e000e610e572099" compaction_input: "e8201779ddf34182ae213e7e493a34c1_SeqId_4_" compaction_input: "ecd1b5bd67844d61a01eb1eced24ed0e_SeqId_4_" compaction_input: "fccc75fd35fa4a3e929246e61857cd50_SeqId_4_" compaction_output: "8aae4544c338479aa1ca6008891e9591" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-18T23:36:09,955 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-18T23:36:09,955 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000008 2024-11-18T23:36:09,957 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/0000000000000000008 2024-11-18T23:36:09,958 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,958 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,960 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:36:09,963 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 739f647798859799f2c1a3c7515c242f 2024-11-18T23:36:09,967 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testCompactedBulkLoadedFiles/739f647798859799f2c1a3c7515c242f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-18T23:36:09,968 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 739f647798859799f2c1a3c7515c242f; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72565167, jitterRate=0.08130525052547455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:36:09,968 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 739f647798859799f2c1a3c7515c242f: Writing region info on filesystem at 1731972969721Initializing all the Stores at 1731972969730 (+9 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972969730Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972969733 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972969733Cleaning up temporary data from old regions at 1731972969958 (+225 ms)Region opened successfully at 1731972969968 (+10 ms) 2024-11-18T23:36:09,972 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 739f647798859799f2c1a3c7515c242f, disabling compactions & flushes 2024-11-18T23:36:09,972 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. 2024-11-18T23:36:09,972 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. 2024-11-18T23:36:09,972 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. after waiting 0 ms 2024-11-18T23:36:09,972 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. 2024-11-18T23:36:09,981 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731972964231.739f647798859799f2c1a3c7515c242f. 2024-11-18T23:36:09,981 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 739f647798859799f2c1a3c7515c242f: Waiting for close lock at 1731972969972Disabling compacts and flushes for region at 1731972969972Disabling writes for close at 1731972969972Writing region close event to WAL at 1731972969980 (+8 ms)Closed at 1731972969980 2024-11-18T23:36:10,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741924_1103 (size=93) 2024-11-18T23:36:10,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741924_1103 (size=93) 2024-11-18T23:36:10,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741924_1103 (size=93) 2024-11-18T23:36:10,037 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:36:10,037 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731972969659) 2024-11-18T23:36:10,062 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=436 (was 430) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:37995 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:37995 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_987990597_22 at /127.0.0.1:37930 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_987990597_22 at /127.0.0.1:50590 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_987990597_22 at /127.0.0.1:36248 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1295 (was 1321), ProcessCount=11 (was 11), AvailableMemoryMB=5780 (was 5876) 2024-11-18T23:36:10,062 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-18T23:36:10,090 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=436, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=1295, ProcessCount=11, AvailableMemoryMB=5777 2024-11-18T23:36:10,090 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-18T23:36:10,116 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:36:10,118 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:36:10,119 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:36:10,128 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77569461, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/hregion-77569461, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:10,147 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77569461/hregion-77569461.1731972970128, exclude list is [], retry=0 2024-11-18T23:36:10,151 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:10,151 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:10,156 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:10,179 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77569461/hregion-77569461.1731972970128 2024-11-18T23:36:10,184 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:36:10,185 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 07baac4669e383e2956c5dfabf91dd5a, NAME => 'testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37995/hbase 2024-11-18T23:36:10,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741926_1105 (size=67) 2024-11-18T23:36:10,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741926_1105 (size=67) 2024-11-18T23:36:10,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741926_1105 (size=67) 2024-11-18T23:36:10,629 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:10,631 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,633 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName a 2024-11-18T23:36:10,633 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:10,634 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:10,634 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,636 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName b 2024-11-18T23:36:10,636 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:10,636 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:10,637 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,638 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName c 2024-11-18T23:36:10,638 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:10,639 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:10,639 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,640 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,645 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,647 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,647 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,648 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:36:10,649 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,654 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:36:10,660 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 07baac4669e383e2956c5dfabf91dd5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64843605, jitterRate=-0.03375498950481415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:36:10,661 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 07baac4669e383e2956c5dfabf91dd5a: Writing region info on filesystem at 1731972970629Initializing all the Stores at 1731972970630 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972970630Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972970631 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972970631Cleaning up temporary data from old regions at 1731972970647 (+16 ms)Region opened successfully at 1731972970661 (+14 ms) 2024-11-18T23:36:10,661 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 07baac4669e383e2956c5dfabf91dd5a, disabling compactions & flushes 2024-11-18T23:36:10,661 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:10,661 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:10,661 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. after waiting 0 ms 2024-11-18T23:36:10,661 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:10,662 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:10,662 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 07baac4669e383e2956c5dfabf91dd5a: Waiting for close lock at 1731972970661Disabling compacts and flushes for region at 1731972970661Disabling writes for close at 1731972970661Writing region close event to WAL at 1731972970662 (+1 ms)Closed at 1731972970662 2024-11-18T23:36:10,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741925_1104 (size=93) 2024-11-18T23:36:10,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741925_1104 (size=93) 2024-11-18T23:36:10,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741925_1104 (size=93) 2024-11-18T23:36:10,694 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:36:10,694 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77569461:(num 1731972970128) 2024-11-18T23:36:10,695 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:36:10,704 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:10,724 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704, exclude list is [], retry=0 2024-11-18T23:36:10,728 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:10,728 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:10,729 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:10,745 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704 2024-11-18T23:36:10,746 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:36:10,746 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 07baac4669e383e2956c5dfabf91dd5a, NAME => 'testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:36:10,747 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:10,747 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,747 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,749 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,750 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName a 2024-11-18T23:36:10,750 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:10,751 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:10,751 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,752 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName b 2024-11-18T23:36:10,753 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:10,756 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:10,756 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,759 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName c 2024-11-18T23:36:10,760 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:10,763 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:10,763 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,768 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,774 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,776 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,776 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,777 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:36:10,779 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:10,780 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 07baac4669e383e2956c5dfabf91dd5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65552384, jitterRate=-0.023193359375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:36:10,781 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 07baac4669e383e2956c5dfabf91dd5a: Writing region info on filesystem at 1731972970747Initializing all the Stores at 1731972970748 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972970748Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972970748Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972970748Cleaning up temporary data from old regions at 1731972970776 (+28 ms)Region opened successfully at 1731972970780 (+4 ms) 2024-11-18T23:36:10,807 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 07baac4669e383e2956c5dfabf91dd5a 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-18T23:36:10,829 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/a/50f7ac55a71344efa5821d4fba5c15b4 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731972970781/Put/seqid=0 2024-11-18T23:36:10,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741928_1107 (size=5958) 2024-11-18T23:36:10,872 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/a/50f7ac55a71344efa5821d4fba5c15b4 2024-11-18T23:36:10,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741928_1107 (size=5958) 2024-11-18T23:36:10,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741928_1107 (size=5958) 2024-11-18T23:36:10,909 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/a/50f7ac55a71344efa5821d4fba5c15b4 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/a/50f7ac55a71344efa5821d4fba5c15b4 2024-11-18T23:36:10,948 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/a/50f7ac55a71344efa5821d4fba5c15b4, entries=10, sequenceid=13, filesize=5.8 K 2024-11-18T23:36:10,955 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 07baac4669e383e2956c5dfabf91dd5a in 149ms, sequenceid=13, compaction requested=false 2024-11-18T23:36:10,955 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 07baac4669e383e2956c5dfabf91dd5a: 2024-11-18T23:36:11,104 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 07baac4669e383e2956c5dfabf91dd5a, disabling compactions & flushes 2024-11-18T23:36:11,104 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:11,104 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:11,104 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. after waiting 0 ms 2024-11-18T23:36:11,104 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:11,112 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:11,113 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:11,113 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 07baac4669e383e2956c5dfabf91dd5a: Waiting for close lock at 1731972971104Disabling compacts and flushes for region at 1731972971104Disabling writes for close at 1731972971104Writing region close event to WAL at 1731972971112 (+8 ms)Closed at 1731972971113 (+1 ms) 2024-11-18T23:36:11,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741927_1106 (size=5869) 2024-11-18T23:36:11,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741927_1106 (size=5869) 2024-11-18T23:36:11,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741927_1106 (size=5869) 2024-11-18T23:36:11,177 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704, size=5.7 K (5869bytes) 2024-11-18T23:36:11,177 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704 2024-11-18T23:36:11,178 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704 after 1ms 2024-11-18T23:36:11,181 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704 took 4ms 2024-11-18T23:36:11,195 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704 so closing down 2024-11-18T23:36:11,195 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:36:11,197 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731972970704.temp 2024-11-18T23:36:11,200 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000003-wal.1731972970704.temp 2024-11-18T23:36:11,202 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:36:11,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741929_1108 (size=5223) 2024-11-18T23:36:11,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741929_1108 (size=5223) 2024-11-18T23:36:11,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741929_1108 (size=5223) 2024-11-18T23:36:11,257 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000003-wal.1731972970704.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-18T23:36:11,265 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000003-wal.1731972970704.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000035 2024-11-18T23:36:11,265 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 73 ms; skipped=2; WAL=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-18T23:36:11,265 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704, journal: Splitting hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704, size=5.7 K (5869bytes) at 1731972971177Finishing writing output for hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704 so closing down at 1731972971195 (+18 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000003-wal.1731972970704.temp at 1731972971200 (+5 ms)3 split writer threads finished at 1731972971202 (+2 ms)Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000003-wal.1731972970704.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731972971257 (+55 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000003-wal.1731972970704.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000035 at 1731972971265 (+8 ms)Processed 32 edits across 1 Regions in 73 ms; skipped=2; WAL=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1731972971265 2024-11-18T23:36:11,268 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972970704 to hdfs://localhost:37995/hbase/oldWALs/wal.1731972970704 2024-11-18T23:36:11,269 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000035 2024-11-18T23:36:11,269 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:36:11,271 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:11,288 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272, exclude list is [], retry=0 2024-11-18T23:36:11,296 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:11,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:11,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:11,317 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 2024-11-18T23:36:11,321 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:36:11,321 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 07baac4669e383e2956c5dfabf91dd5a, NAME => 'testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:36:11,322 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:11,322 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,322 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,324 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,329 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName a 2024-11-18T23:36:11,329 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:11,345 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/a/50f7ac55a71344efa5821d4fba5c15b4 2024-11-18T23:36:11,345 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:11,345 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,346 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName b 2024-11-18T23:36:11,347 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:11,347 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:11,347 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,351 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName c 2024-11-18T23:36:11,351 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:11,352 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:11,352 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,353 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,356 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,361 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000035 2024-11-18T23:36:11,370 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000035 2024-11-18T23:36:11,370 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 07baac4669e383e2956c5dfabf91dd5a 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-18T23:36:11,392 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/b/4b585066b8734785bea954b51d218c75 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731972970955/Put/seqid=0 2024-11-18T23:36:11,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741931_1110 (size=5958) 2024-11-18T23:36:11,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741931_1110 (size=5958) 2024-11-18T23:36:11,424 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/b/4b585066b8734785bea954b51d218c75 2024-11-18T23:36:11,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741931_1110 (size=5958) 2024-11-18T23:36:11,462 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/c/952edf8340204e7286be456ea4296604 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731972971028/Put/seqid=0 2024-11-18T23:36:11,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741932_1111 (size=5958) 2024-11-18T23:36:11,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741932_1111 (size=5958) 2024-11-18T23:36:11,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741932_1111 (size=5958) 2024-11-18T23:36:11,881 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/c/952edf8340204e7286be456ea4296604 2024-11-18T23:36:11,889 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/b/4b585066b8734785bea954b51d218c75 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/b/4b585066b8734785bea954b51d218c75 2024-11-18T23:36:11,897 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/b/4b585066b8734785bea954b51d218c75, entries=10, sequenceid=35, filesize=5.8 K 2024-11-18T23:36:11,899 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/c/952edf8340204e7286be456ea4296604 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/c/952edf8340204e7286be456ea4296604 2024-11-18T23:36:11,907 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/c/952edf8340204e7286be456ea4296604, entries=10, sequenceid=35, filesize=5.8 K 2024-11-18T23:36:11,908 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 07baac4669e383e2956c5dfabf91dd5a in 538ms, sequenceid=35, compaction requested=false; wal=null 2024-11-18T23:36:11,912 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000035 2024-11-18T23:36:11,915 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,915 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,917 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:36:11,921 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:11,925 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-18T23:36:11,930 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 07baac4669e383e2956c5dfabf91dd5a; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71754865, jitterRate=0.0692308098077774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:36:11,930 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 07baac4669e383e2956c5dfabf91dd5a: Writing region info on filesystem at 1731972971322Initializing all the Stores at 1731972971323 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972971323Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972971324 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972971324Obtaining lock to block concurrent updates at 1731972971370 (+46 ms)Preparing flush snapshotting stores in 07baac4669e383e2956c5dfabf91dd5a at 1731972971370Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731972971371 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. at 1731972971371Flushing 07baac4669e383e2956c5dfabf91dd5a/b: creating writer at 1731972971371Flushing 07baac4669e383e2956c5dfabf91dd5a/b: appending metadata at 1731972971390 (+19 ms)Flushing 07baac4669e383e2956c5dfabf91dd5a/b: closing flushed file at 1731972971390Flushing 07baac4669e383e2956c5dfabf91dd5a/c: creating writer at 1731972971442 (+52 ms)Flushing 07baac4669e383e2956c5dfabf91dd5a/c: appending metadata at 1731972971461 (+19 ms)Flushing 07baac4669e383e2956c5dfabf91dd5a/c: closing flushed file at 1731972971461Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9da6175: reopening flushed file at 1731972971888 (+427 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35b831f0: reopening flushed file at 1731972971898 (+10 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 07baac4669e383e2956c5dfabf91dd5a in 538ms, sequenceid=35, compaction requested=false; wal=null at 1731972971908 (+10 ms)Cleaning up temporary data from old regions at 1731972971915 (+7 ms)Region opened successfully at 1731972971930 (+15 ms) 2024-11-18T23:36:12,098 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272, size=0 (0bytes) 2024-11-18T23:36:12,098 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 might be still open, length is 0 2024-11-18T23:36:12,098 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 2024-11-18T23:36:12,098 WARN [IPC Server handler 0 on default port 37995 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-18T23:36:12,098 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 after 0ms 2024-11-18T23:36:13,406 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:50624 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50624 dst: /127.0.0.1:36973 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36973 remote=/127.0.0.1:50624]. Total timeout mills is 60000, 58621 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:13,406 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:38000 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:39707:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38000 dst: /127.0.0.1:39707 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:13,407 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-904604702_22 at /127.0.0.1:36302 [Receiving block BP-330010681-172.17.0.2-1731972917172:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:35345:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36302 dst: /127.0.0.1:35345 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:13,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741930_1112 (size=5215) 2024-11-18T23:36:13,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741930_1112 (size=5215) 2024-11-18T23:36:13,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741930_1112 (size=5215) 2024-11-18T23:36:13,916 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-18T23:36:13,916 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-18T23:36:13,917 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-18T23:36:13,917 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-18T23:36:16,099 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 after 4001ms 2024-11-18T23:36:16,102 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 took 4005ms 2024-11-18T23:36:16,104 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272; continuing. 2024-11-18T23:36:16,104 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 so closing down 2024-11-18T23:36:16,104 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-18T23:36:16,105 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731972971272.temp 2024-11-18T23:36:16,107 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000037-wal.1731972971272.temp 2024-11-18T23:36:16,107 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-18T23:36:16,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741933_1113 (size=5223) 2024-11-18T23:36:16,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741933_1113 (size=5223) 2024-11-18T23:36:16,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741933_1113 (size=5223) 2024-11-18T23:36:16,114 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000037-wal.1731972971272.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-18T23:36:16,118 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000037-wal.1731972971272.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000066 2024-11-18T23:36:16,118 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272, size=0, length=0, corrupted=false, cancelled=false 2024-11-18T23:36:16,118 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272, journal: Splitting hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272, size=0 (0bytes) at 1731972972098Finishing writing output for hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 so closing down at 1731972976104 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000037-wal.1731972971272.temp at 1731972976107 (+3 ms)3 split writer threads finished at 1731972976107Closed recovered edits writer path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000037-wal.1731972971272.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731972976114 (+7 ms)Rename recovered edits hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000037-wal.1731972971272.temp to hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000066 at 1731972976118 (+4 ms)Processed 30 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272, size=0, length=0, corrupted=false, cancelled=false at 1731972976118 2024-11-18T23:36:16,120 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 to hdfs://localhost:37995/hbase/oldWALs/wal.1731972971272 2024-11-18T23:36:16,123 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000066 2024-11-18T23:36:16,123 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-18T23:36:16,125 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37995/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114, archiveDir=hdfs://localhost:37995/hbase/oldWALs, maxLogs=32 2024-11-18T23:36:16,144 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972976126, exclude list is [], retry=0 2024-11-18T23:36:16,147 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39707,DS-bd9a80b9-d9ca-453c-916a-d97a723d440f,DISK] 2024-11-18T23:36:16,147 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36973,DS-abd90c94-af46-4db3-8b6f-f6a6466c166d,DISK] 2024-11-18T23:36:16,147 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35345,DS-56362478-04dc-42c8-8ca6-fdeeb4b63d03,DISK] 2024-11-18T23:36:16,152 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972976126 2024-11-18T23:36:16,152 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46723:46723),(127.0.0.1/127.0.0.1:42157:42157),(127.0.0.1/127.0.0.1:42361:42361)] 2024-11-18T23:36:16,153 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:36:16,156 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:16,157 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName a 2024-11-18T23:36:16,157 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:16,171 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/a/50f7ac55a71344efa5821d4fba5c15b4 2024-11-18T23:36:16,171 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:16,171 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:16,173 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName b 2024-11-18T23:36:16,173 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:16,192 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/b/4b585066b8734785bea954b51d218c75 2024-11-18T23:36:16,192 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:16,192 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:16,194 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07baac4669e383e2956c5dfabf91dd5a columnFamilyName c 2024-11-18T23:36:16,194 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:36:16,207 DEBUG [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/c/952edf8340204e7286be456ea4296604 2024-11-18T23:36:16,208 INFO [StoreOpener-07baac4669e383e2956c5dfabf91dd5a-1 {}] regionserver.HStore(327): Store=07baac4669e383e2956c5dfabf91dd5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:36:16,211 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:16,216 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:16,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:16,225 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000066 2024-11-18T23:36:16,232 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000066 2024-11-18T23:36:16,233 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 07baac4669e383e2956c5dfabf91dd5a 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-18T23:36:16,270 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/a/bea24739407d434db751cfd861ea5791 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731972971943/Put/seqid=0 2024-11-18T23:36:16,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741935_1115 (size=5958) 2024-11-18T23:36:16,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741935_1115 (size=5958) 2024-11-18T23:36:16,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741935_1115 (size=5958) 2024-11-18T23:36:16,338 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/a/bea24739407d434db751cfd861ea5791 2024-11-18T23:36:16,399 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/b/71cb0ce5397e4d65b72ba1c1526632f0 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731972971967/Put/seqid=0 2024-11-18T23:36:16,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741936_1116 (size=5958) 2024-11-18T23:36:16,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741936_1116 (size=5958) 2024-11-18T23:36:16,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741936_1116 (size=5958) 2024-11-18T23:36:16,436 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/b/71cb0ce5397e4d65b72ba1c1526632f0 2024-11-18T23:36:16,470 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/c/36a9c181420c4fdf997d1616dd49867e is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731972971992/Put/seqid=0 2024-11-18T23:36:16,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741937_1117 (size=5958) 2024-11-18T23:36:16,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741937_1117 (size=5958) 2024-11-18T23:36:16,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741937_1117 (size=5958) 2024-11-18T23:36:16,487 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/c/36a9c181420c4fdf997d1616dd49867e 2024-11-18T23:36:16,495 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/a/bea24739407d434db751cfd861ea5791 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/a/bea24739407d434db751cfd861ea5791 2024-11-18T23:36:16,500 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/a/bea24739407d434db751cfd861ea5791, entries=10, sequenceid=66, filesize=5.8 K 2024-11-18T23:36:16,501 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/b/71cb0ce5397e4d65b72ba1c1526632f0 as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/b/71cb0ce5397e4d65b72ba1c1526632f0 2024-11-18T23:36:16,506 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/b/71cb0ce5397e4d65b72ba1c1526632f0, entries=10, sequenceid=66, filesize=5.8 K 2024-11-18T23:36:16,507 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/.tmp/c/36a9c181420c4fdf997d1616dd49867e as hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/c/36a9c181420c4fdf997d1616dd49867e 2024-11-18T23:36:16,513 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/c/36a9c181420c4fdf997d1616dd49867e, entries=10, sequenceid=66, filesize=5.8 K 2024-11-18T23:36:16,513 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 07baac4669e383e2956c5dfabf91dd5a in 281ms, sequenceid=66, compaction requested=false; wal=null 2024-11-18T23:36:16,515 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/0000000000000000066 2024-11-18T23:36:16,516 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:16,516 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:16,517 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T23:36:16,518 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 07baac4669e383e2956c5dfabf91dd5a 2024-11-18T23:36:16,520 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/hbase/data/default/testReplayEditsWrittenViaHRegion/07baac4669e383e2956c5dfabf91dd5a/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-18T23:36:16,521 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 07baac4669e383e2956c5dfabf91dd5a; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69005812, jitterRate=0.028266727924346924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T23:36:16,522 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 07baac4669e383e2956c5dfabf91dd5a: Writing region info on filesystem at 1731972976153Initializing all the Stores at 1731972976154 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972976154Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972976156 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731972976156Obtaining lock to block concurrent updates at 1731972976233 (+77 ms)Preparing flush snapshotting stores in 07baac4669e383e2956c5dfabf91dd5a at 1731972976233Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731972976233Flushing stores of testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. at 1731972976233Flushing 07baac4669e383e2956c5dfabf91dd5a/a: creating writer at 1731972976233Flushing 07baac4669e383e2956c5dfabf91dd5a/a: appending metadata at 1731972976269 (+36 ms)Flushing 07baac4669e383e2956c5dfabf91dd5a/a: closing flushed file at 1731972976269Flushing 07baac4669e383e2956c5dfabf91dd5a/b: creating writer at 1731972976369 (+100 ms)Flushing 07baac4669e383e2956c5dfabf91dd5a/b: appending metadata at 1731972976398 (+29 ms)Flushing 07baac4669e383e2956c5dfabf91dd5a/b: closing flushed file at 1731972976398Flushing 07baac4669e383e2956c5dfabf91dd5a/c: creating writer at 1731972976446 (+48 ms)Flushing 07baac4669e383e2956c5dfabf91dd5a/c: appending metadata at 1731972976469 (+23 ms)Flushing 07baac4669e383e2956c5dfabf91dd5a/c: closing flushed file at 1731972976470 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62dfb7b8: reopening flushed file at 1731972976494 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ad78bd6: reopening flushed file at 1731972976500 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6100bedd: reopening flushed file at 1731972976506 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 07baac4669e383e2956c5dfabf91dd5a in 281ms, sequenceid=66, compaction requested=false; wal=null at 1731972976513 (+7 ms)Cleaning up temporary data from old regions at 1731972976516 (+3 ms)Region opened successfully at 1731972976521 (+5 ms) 2024-11-18T23:36:16,543 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 07baac4669e383e2956c5dfabf91dd5a, disabling compactions & flushes 2024-11-18T23:36:16,543 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:16,543 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:16,543 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. after waiting 0 ms 2024-11-18T23:36:16,544 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:16,545 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731972970116.07baac4669e383e2956c5dfabf91dd5a. 2024-11-18T23:36:16,545 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 07baac4669e383e2956c5dfabf91dd5a: Waiting for close lock at 1731972976543Disabling compacts and flushes for region at 1731972976543Disabling writes for close at 1731972976543Writing region close event to WAL at 1731972976545 (+2 ms)Closed at 1731972976545 2024-11-18T23:36:16,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741934_1114 (size=93) 2024-11-18T23:36:16,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741934_1114 (size=93) 2024-11-18T23:36:16,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741934_1114 (size=93) 2024-11-18T23:36:16,551 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-18T23:36:16,551 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731972976126) 2024-11-18T23:36:16,569 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443 (was 436) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1004534962) connection to localhost/127.0.0.1:37995 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1391854750_22 at /127.0.0.1:40048 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:37995 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1391854750_22 at /127.0.0.1:45500 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1391854750_22 at /127.0.0.1:41444 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1406 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1267 (was 1295), ProcessCount=11 (was 11), AvailableMemoryMB=5760 (was 5777) 2024-11-18T23:36:16,569 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1406 is superior to 1024 2024-11-18T23:36:16,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T23:36:16,570 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:36:16,570 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:36:16,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:36:16,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:36:16,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:36:16,571 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T23:36:16,571 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=818882006, stopped=false 2024-11-18T23:36:16,572 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9d96596e55a9,40599,1731972923486 2024-11-18T23:36:16,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:36:16,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:36:16,584 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T23:36:16,584 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:36:16,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:36:16,584 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:36:16,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:36:16,585 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:36:16,586 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:36:16,586 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:36:16,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:36:16,586 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9d96596e55a9,33655,1731972924943' ***** 2024-11-18T23:36:16,586 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T23:36:16,586 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T23:36:16,586 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9d96596e55a9,46043,1731972925008' ***** 2024-11-18T23:36:16,586 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T23:36:16,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:36:16,587 INFO [RS:1;9d96596e55a9:33655 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T23:36:16,587 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T23:36:16,587 INFO [RS:1;9d96596e55a9:33655 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T23:36:16,587 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(959): stopping server 9d96596e55a9,33655,1731972924943 2024-11-18T23:36:16,587 INFO [RS:1;9d96596e55a9:33655 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:36:16,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:36:16,587 INFO [RS:1;9d96596e55a9:33655 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9d96596e55a9:33655. 2024-11-18T23:36:16,587 DEBUG [RS:1;9d96596e55a9:33655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:36:16,587 DEBUG [RS:1;9d96596e55a9:33655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:36:16,587 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T23:36:16,587 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T23:36:16,587 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T23:36:16,587 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T23:36:16,587 INFO [RS:2;9d96596e55a9:46043 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T23:36:16,587 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T23:36:16,587 INFO [RS:2;9d96596e55a9:46043 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T23:36:16,587 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T23:36:16,587 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(3091): Received CLOSE for b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:36:16,588 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T23:36:16,588 DEBUG [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-18T23:36:16,588 DEBUG [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T23:36:16,588 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:36:16,588 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:36:16,588 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:36:16,588 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:36:16,588 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:36:16,588 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-18T23:36:16,592 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(959): stopping server 9d96596e55a9,46043,1731972925008 2024-11-18T23:36:16,592 INFO [RS:2;9d96596e55a9:46043 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:36:16,592 INFO [RS:2;9d96596e55a9:46043 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9d96596e55a9:46043. 2024-11-18T23:36:16,592 DEBUG [RS:2;9d96596e55a9:46043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:36:16,592 DEBUG [RS:2;9d96596e55a9:46043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:36:16,593 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T23:36:16,593 DEBUG [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(1325): Online Regions={b890df8ea0d9342b29ca822fc6dfed52=testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52.} 2024-11-18T23:36:16,593 DEBUG [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(1351): Waiting on b890df8ea0d9342b29ca822fc6dfed52 2024-11-18T23:36:16,596 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b890df8ea0d9342b29ca822fc6dfed52, disabling compactions & flushes 2024-11-18T23:36:16,596 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:36:16,597 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:36:16,597 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. after waiting 0 ms 2024-11-18T23:36:16,597 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:36:16,628 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b890df8ea0d9342b29ca822fc6dfed52/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-18T23:36:16,629 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:36:16,629 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b890df8ea0d9342b29ca822fc6dfed52: Waiting for close lock at 1731972976596Running coprocessor pre-close hooks at 1731972976596Disabling compacts and flushes for region at 1731972976596Disabling writes for close at 1731972976597 (+1 ms)Writing region close event to WAL at 1731972976607 (+10 ms)Running coprocessor post-close hooks at 1731972976628 (+21 ms)Closed at 1731972976628 2024-11-18T23:36:16,629 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52. 2024-11-18T23:36:16,635 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/.tmp/info/5960abf2b51440b58fd45a09a5d41206 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731972945954.b890df8ea0d9342b29ca822fc6dfed52./info:regioninfo/1731972950251/Put/seqid=0 2024-11-18T23:36:16,640 INFO [regionserver/9d96596e55a9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:36:16,652 INFO [regionserver/9d96596e55a9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:36:16,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741938_1118 (size=8243) 2024-11-18T23:36:16,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741938_1118 (size=8243) 2024-11-18T23:36:16,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741938_1118 (size=8243) 2024-11-18T23:36:16,662 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/.tmp/info/5960abf2b51440b58fd45a09a5d41206 2024-11-18T23:36:16,690 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/.tmp/ns/5cc5fee9a0fe4efbb83661976aad3446 is 43, key is default/ns:d/1731972928971/Put/seqid=0 2024-11-18T23:36:16,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741939_1119 (size=5153) 2024-11-18T23:36:16,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741939_1119 (size=5153) 2024-11-18T23:36:16,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741939_1119 (size=5153) 2024-11-18T23:36:16,705 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/.tmp/ns/5cc5fee9a0fe4efbb83661976aad3446 2024-11-18T23:36:16,740 INFO [regionserver/9d96596e55a9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T23:36:16,740 INFO [regionserver/9d96596e55a9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T23:36:16,742 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/.tmp/table/0aa14074b8fd4e8fbad1507db01d4d48 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731972946490/Put/seqid=0 2024-11-18T23:36:16,775 INFO [regionserver/9d96596e55a9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T23:36:16,775 INFO [regionserver/9d96596e55a9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T23:36:16,788 DEBUG [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T23:36:16,793 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(976): stopping server 9d96596e55a9,46043,1731972925008; all regions closed. 2024-11-18T23:36:16,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741940_1120 (size=5431) 2024-11-18T23:36:16,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741940_1120 (size=5431) 2024-11-18T23:36:16,808 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/.tmp/table/0aa14074b8fd4e8fbad1507db01d4d48 2024-11-18T23:36:16,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741940_1120 (size=5431) 2024-11-18T23:36:16,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741834_1010 (size=928) 2024-11-18T23:36:16,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741834_1010 (size=928) 2024-11-18T23:36:16,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741834_1010 (size=928) 2024-11-18T23:36:16,814 DEBUG [RS:2;9d96596e55a9:46043 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/oldWALs 2024-11-18T23:36:16,814 INFO [RS:2;9d96596e55a9:46043 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9d96596e55a9%2C46043%2C1731972925008:(num 1731972927993) 2024-11-18T23:36:16,814 DEBUG [RS:2;9d96596e55a9:46043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:36:16,815 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:36:16,815 INFO [RS:2;9d96596e55a9:46043 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:36:16,815 INFO [RS:2;9d96596e55a9:46043 {}] hbase.ChoreService(370): Chore service for: regionserver/9d96596e55a9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T23:36:16,815 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T23:36:16,815 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T23:36:16,815 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T23:36:16,815 INFO [RS:2;9d96596e55a9:46043 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:36:16,816 INFO [regionserver/9d96596e55a9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:36:16,817 INFO [RS:2;9d96596e55a9:46043 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46043 2024-11-18T23:36:16,827 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/.tmp/info/5960abf2b51440b58fd45a09a5d41206 as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/info/5960abf2b51440b58fd45a09a5d41206 2024-11-18T23:36:16,844 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/info/5960abf2b51440b58fd45a09a5d41206, entries=18, sequenceid=21, filesize=8.0 K 2024-11-18T23:36:16,847 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/.tmp/ns/5cc5fee9a0fe4efbb83661976aad3446 as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/ns/5cc5fee9a0fe4efbb83661976aad3446 2024-11-18T23:36:16,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:36:16,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9d96596e55a9,46043,1731972925008 2024-11-18T23:36:16,853 INFO [RS:2;9d96596e55a9:46043 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:36:16,862 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/ns/5cc5fee9a0fe4efbb83661976aad3446, entries=2, sequenceid=21, filesize=5.0 K 2024-11-18T23:36:16,865 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/.tmp/table/0aa14074b8fd4e8fbad1507db01d4d48 as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/table/0aa14074b8fd4e8fbad1507db01d4d48 2024-11-18T23:36:16,871 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/table/0aa14074b8fd4e8fbad1507db01d4d48, entries=2, sequenceid=21, filesize=5.3 K 2024-11-18T23:36:16,901 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9d96596e55a9,46043,1731972925008] 2024-11-18T23:36:16,901 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 313ms, sequenceid=21, compaction requested=false 2024-11-18T23:36:16,909 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-18T23:36:16,912 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T23:36:16,912 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:36:16,912 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731972976588Running coprocessor pre-close hooks at 1731972976588Disabling compacts and flushes for region at 1731972976588Disabling writes for close at 1731972976588Obtaining lock to block concurrent updates at 1731972976588Preparing flush snapshotting stores in 1588230740 at 1731972976588Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731972976589 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731972976589Flushing 1588230740/info: creating writer at 1731972976590 (+1 ms)Flushing 1588230740/info: appending metadata at 1731972976634 (+44 ms)Flushing 1588230740/info: closing flushed file at 1731972976634Flushing 1588230740/ns: creating writer at 1731972976670 (+36 ms)Flushing 1588230740/ns: appending metadata at 1731972976690 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731972976690Flushing 1588230740/table: creating writer at 1731972976717 (+27 ms)Flushing 1588230740/table: appending metadata at 1731972976741 (+24 ms)Flushing 1588230740/table: closing flushed file at 1731972976741Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@448e042f: reopening flushed file at 1731972976824 (+83 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@736cdb06: reopening flushed file at 1731972976845 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5659b953: reopening flushed file at 1731972976864 (+19 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 313ms, sequenceid=21, compaction requested=false at 1731972976902 (+38 ms)Writing region close event to WAL at 1731972976903 (+1 ms)Running coprocessor post-close hooks at 1731972976912 (+9 ms)Closed at 1731972976912 2024-11-18T23:36:16,913 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T23:36:16,964 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9d96596e55a9,46043,1731972925008 already deleted, retry=false 2024-11-18T23:36:16,964 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9d96596e55a9,46043,1731972925008 expired; onlineServers=1 2024-11-18T23:36:16,989 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(976): stopping server 9d96596e55a9,33655,1731972924943; all regions closed. 2024-11-18T23:36:16,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741836_1012 (size=8899) 2024-11-18T23:36:16,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741836_1012 (size=8899) 2024-11-18T23:36:16,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741836_1012 (size=8899) 2024-11-18T23:36:16,994 DEBUG [RS:1;9d96596e55a9:33655 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/oldWALs 2024-11-18T23:36:16,994 INFO [RS:1;9d96596e55a9:33655 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9d96596e55a9%2C33655%2C1731972924943.meta:.meta(num 1731972928597) 2024-11-18T23:36:16,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741833_1009 (size=2598) 2024-11-18T23:36:16,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741833_1009 (size=2598) 2024-11-18T23:36:16,998 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/WALs/9d96596e55a9,33655,1731972924943/9d96596e55a9%2C33655%2C1731972924943.1731972927957 not finished, retry = 0 2024-11-18T23:36:16,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741833_1009 (size=2598) 2024-11-18T23:36:17,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:36:17,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46043-0x101504d78350003, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:36:17,001 INFO [RS:2;9d96596e55a9:46043 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:36:17,001 INFO [RS:2;9d96596e55a9:46043 {}] regionserver.HRegionServer(1031): Exiting; stopping=9d96596e55a9,46043,1731972925008; zookeeper connection closed. 2024-11-18T23:36:17,004 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ae884b4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ae884b4 2024-11-18T23:36:17,101 DEBUG [RS:1;9d96596e55a9:33655 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/oldWALs 2024-11-18T23:36:17,101 INFO [RS:1;9d96596e55a9:33655 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9d96596e55a9%2C33655%2C1731972924943:(num 1731972927957) 2024-11-18T23:36:17,101 DEBUG [RS:1;9d96596e55a9:33655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:36:17,101 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:36:17,102 INFO [RS:1;9d96596e55a9:33655 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:36:17,102 INFO [RS:1;9d96596e55a9:33655 {}] hbase.ChoreService(370): Chore service for: regionserver/9d96596e55a9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T23:36:17,102 INFO [RS:1;9d96596e55a9:33655 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:36:17,102 INFO [regionserver/9d96596e55a9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:36:17,102 INFO [RS:1;9d96596e55a9:33655 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33655 2024-11-18T23:36:17,110 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9d96596e55a9,33655,1731972924943 2024-11-18T23:36:17,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:36:17,111 INFO [RS:1;9d96596e55a9:33655 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:36:17,111 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9d96596e55a9,33655,1731972924943] 2024-11-18T23:36:17,139 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9d96596e55a9,33655,1731972924943 already deleted, retry=false 2024-11-18T23:36:17,139 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9d96596e55a9,33655,1731972924943 expired; onlineServers=0 2024-11-18T23:36:17,139 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9d96596e55a9,40599,1731972923486' ***** 2024-11-18T23:36:17,139 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T23:36:17,139 INFO [M:0;9d96596e55a9:40599 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:36:17,139 INFO [M:0;9d96596e55a9:40599 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:36:17,139 DEBUG [M:0;9d96596e55a9:40599 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T23:36:17,140 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T23:36:17,140 DEBUG [M:0;9d96596e55a9:40599 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T23:36:17,140 DEBUG [master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.large.0-1731972927262 {}] cleaner.HFileCleaner(306): Exit Thread[master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.large.0-1731972927262,5,FailOnTimeoutGroup] 2024-11-18T23:36:17,140 DEBUG [master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.small.0-1731972927265 {}] cleaner.HFileCleaner(306): Exit Thread[master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.small.0-1731972927265,5,FailOnTimeoutGroup] 2024-11-18T23:36:17,140 INFO [M:0;9d96596e55a9:40599 {}] hbase.ChoreService(370): Chore service for: master/9d96596e55a9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T23:36:17,140 INFO [M:0;9d96596e55a9:40599 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:36:17,140 DEBUG [M:0;9d96596e55a9:40599 {}] master.HMaster(1795): Stopping service threads 2024-11-18T23:36:17,140 INFO [M:0;9d96596e55a9:40599 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T23:36:17,140 INFO [M:0;9d96596e55a9:40599 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T23:36:17,141 INFO [M:0;9d96596e55a9:40599 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T23:36:17,141 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T23:36:17,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T23:36:17,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:36:17,160 DEBUG [M:0;9d96596e55a9:40599 {}] zookeeper.ZKUtil(347): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T23:36:17,160 WARN [M:0;9d96596e55a9:40599 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T23:36:17,161 INFO [M:0;9d96596e55a9:40599 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/.lastflushedseqids 2024-11-18T23:36:17,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741941_1121 (size=138) 2024-11-18T23:36:17,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741941_1121 (size=138) 2024-11-18T23:36:17,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741941_1121 (size=138) 2024-11-18T23:36:17,221 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:36:17,221 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33655-0x101504d78350002, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:36:17,221 INFO [RS:1;9d96596e55a9:33655 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:36:17,221 INFO [RS:1;9d96596e55a9:33655 {}] regionserver.HRegionServer(1031): Exiting; stopping=9d96596e55a9,33655,1731972924943; zookeeper connection closed. 2024-11-18T23:36:17,221 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3f26673c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3f26673c 2024-11-18T23:36:17,222 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-18T23:36:17,579 INFO [M:0;9d96596e55a9:40599 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T23:36:17,580 INFO [M:0;9d96596e55a9:40599 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T23:36:17,580 DEBUG [M:0;9d96596e55a9:40599 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T23:36:17,580 INFO [M:0;9d96596e55a9:40599 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:36:17,580 DEBUG [M:0;9d96596e55a9:40599 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:36:17,580 DEBUG [M:0;9d96596e55a9:40599 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T23:36:17,580 DEBUG [M:0;9d96596e55a9:40599 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:36:17,580 INFO [M:0;9d96596e55a9:40599 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-18T23:36:17,598 DEBUG [M:0;9d96596e55a9:40599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/87ce7aee0c2d4cae9a536f7e8f1f83ba is 82, key is hbase:meta,,1/info:regioninfo/1731972928767/Put/seqid=0 2024-11-18T23:36:17,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741942_1122 (size=5672) 2024-11-18T23:36:17,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741942_1122 (size=5672) 2024-11-18T23:36:17,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741942_1122 (size=5672) 2024-11-18T23:36:17,607 INFO [M:0;9d96596e55a9:40599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/87ce7aee0c2d4cae9a536f7e8f1f83ba 2024-11-18T23:36:17,636 DEBUG [M:0;9d96596e55a9:40599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc5602e98971457dbc0db5789429f5c9 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731972946502/Put/seqid=0 2024-11-18T23:36:17,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741943_1123 (size=7755) 2024-11-18T23:36:17,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741943_1123 (size=7755) 2024-11-18T23:36:17,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741943_1123 (size=7755) 2024-11-18T23:36:18,075 INFO [M:0;9d96596e55a9:40599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc5602e98971457dbc0db5789429f5c9 2024-11-18T23:36:18,082 INFO [M:0;9d96596e55a9:40599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fc5602e98971457dbc0db5789429f5c9 2024-11-18T23:36:18,105 DEBUG [M:0;9d96596e55a9:40599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c000c3798cba43eab83b05ad184d86f4 is 69, key is 9d96596e55a9,33655,1731972924943/rs:state/1731972927355/Put/seqid=0 2024-11-18T23:36:18,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741944_1124 (size=5440) 2024-11-18T23:36:18,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741944_1124 (size=5440) 2024-11-18T23:36:18,155 INFO [M:0;9d96596e55a9:40599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c000c3798cba43eab83b05ad184d86f4 2024-11-18T23:36:18,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741944_1124 (size=5440) 2024-11-18T23:36:18,171 INFO [M:0;9d96596e55a9:40599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c000c3798cba43eab83b05ad184d86f4 2024-11-18T23:36:18,173 DEBUG [M:0;9d96596e55a9:40599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/87ce7aee0c2d4cae9a536f7e8f1f83ba as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/87ce7aee0c2d4cae9a536f7e8f1f83ba 2024-11-18T23:36:18,197 INFO [M:0;9d96596e55a9:40599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/87ce7aee0c2d4cae9a536f7e8f1f83ba, entries=8, sequenceid=168, filesize=5.5 K 2024-11-18T23:36:18,199 DEBUG [M:0;9d96596e55a9:40599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc5602e98971457dbc0db5789429f5c9 as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fc5602e98971457dbc0db5789429f5c9 2024-11-18T23:36:18,227 INFO [M:0;9d96596e55a9:40599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fc5602e98971457dbc0db5789429f5c9 2024-11-18T23:36:18,228 INFO [M:0;9d96596e55a9:40599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fc5602e98971457dbc0db5789429f5c9, entries=17, sequenceid=168, filesize=7.6 K 2024-11-18T23:36:18,233 DEBUG [M:0;9d96596e55a9:40599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c000c3798cba43eab83b05ad184d86f4 as hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c000c3798cba43eab83b05ad184d86f4 2024-11-18T23:36:18,259 INFO [M:0;9d96596e55a9:40599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c000c3798cba43eab83b05ad184d86f4 2024-11-18T23:36:18,259 INFO [M:0;9d96596e55a9:40599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37995/user/jenkins/test-data/0ce4c503-36f5-f1a9-4378-e28cf29bac5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c000c3798cba43eab83b05ad184d86f4, entries=3, sequenceid=168, filesize=5.3 K 2024-11-18T23:36:18,263 INFO [M:0;9d96596e55a9:40599 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 682ms, sequenceid=168, compaction requested=false 2024-11-18T23:36:18,269 INFO [M:0;9d96596e55a9:40599 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:36:18,269 DEBUG [M:0;9d96596e55a9:40599 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731972977580Disabling compacts and flushes for region at 1731972977580Disabling writes for close at 1731972977580Obtaining lock to block concurrent updates at 1731972977580Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731972977580Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1731972977581 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731972977582 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731972977582Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731972977597 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731972977598 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731972977615 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731972977636 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731972977636Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731972978082 (+446 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731972978104 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731972978104Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12ae6883: reopening flushed file at 1731972978171 (+67 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@526fa72e: reopening flushed file at 1731972978198 (+27 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c73b670: reopening flushed file at 1731972978228 (+30 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 682ms, sequenceid=168, compaction requested=false at 1731972978263 (+35 ms)Writing region close event to WAL at 1731972978269 (+6 ms)Closed at 1731972978269 2024-11-18T23:36:18,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36973 is added to blk_1073741830_1006 (size=81431) 2024-11-18T23:36:18,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35345 is added to blk_1073741830_1006 (size=81431) 2024-11-18T23:36:18,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39707 is added to blk_1073741830_1006 (size=81431) 2024-11-18T23:36:18,282 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:36:18,282 INFO [M:0;9d96596e55a9:40599 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T23:36:18,283 INFO [M:0;9d96596e55a9:40599 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40599 2024-11-18T23:36:18,283 INFO [M:0;9d96596e55a9:40599 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:36:18,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:36:18,390 INFO [M:0;9d96596e55a9:40599 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:36:18,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x101504d78350000, quorum=127.0.0.1:59409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:36:18,400 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951584 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951584 (inode 16655) Holder DFSClient_NONMAPREDUCE_-904604702_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731972951081/wal.1731972951584 (inode 16655) Holder DFSClient_NONMAPREDUCE_-904604702_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-18T23:36:18,401 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731972941104/wal.1731972941257 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:18,402 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731972970114/wal.1731972971272 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:18,409 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972961756 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972961756 (inode 16678) Holder DFSClient_NONMAPREDUCE_-904604702_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731972951932/wal.1731972961756 (inode 16678) Holder DFSClient_NONMAPREDUCE_-904604702_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-18T23:36:18,410 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731972931631/wal.1731972931772 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:18,422 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963913 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963913 (inode 16704) Holder DFSClient_NONMAPREDUCE_-904604702_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731972962619/wal.1731972963913 (inode 16704) Holder DFSClient_NONMAPREDUCE_-904604702_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-18T23:36:18,423 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731972964227/wal.1731972964852 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:36:18,429 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal.1731972931381 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal.1731972931381 (inode 16485) Holder DFSClient_NONMAPREDUCE_-904604702_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731972930731/wal.1731972931381 (inode 16485) Holder DFSClient_NONMAPREDUCE_-904604702_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-18T23:36:18,444 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal.1731972930404 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal.1731972930404 (inode 16462) Holder DFSClient_NONMAPREDUCE_-904604702_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731972929376/wal.1731972930404 (inode 16462) Holder DFSClient_NONMAPREDUCE_-904604702_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-18T23:36:18,460 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ed6f3ed{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T23:36:18,464 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f154eb3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T23:36:18,464 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T23:36:18,464 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@147d1a17{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T23:36:18,464 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@626fc3bb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/hadoop.log.dir/,STOPPED} 2024-11-18T23:36:18,472 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T23:36:18,472 WARN [BP-330010681-172.17.0.2-1731972917172 heartbeating to localhost/127.0.0.1:37995 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T23:36:18,473 WARN [BP-330010681-172.17.0.2-1731972917172 heartbeating to localhost/127.0.0.1:37995 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-330010681-172.17.0.2-1731972917172 (Datanode Uuid a3b93d3a-3bd9-42e0-ba1e-40787e0210fd) service to localhost/127.0.0.1:37995 2024-11-18T23:36:18,473 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T23:36:18,474 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data5/current/BP-330010681-172.17.0.2-1731972917172 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T23:36:18,474 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data6/current/BP-330010681-172.17.0.2-1731972917172 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T23:36:18,476 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T23:36:18,482 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4cc5338b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T23:36:18,483 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1364e1fb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T23:36:18,483 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T23:36:18,483 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4925ca3e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T23:36:18,483 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2599967d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/hadoop.log.dir/,STOPPED} 2024-11-18T23:36:18,485 WARN [BP-330010681-172.17.0.2-1731972917172 heartbeating to localhost/127.0.0.1:37995 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T23:36:18,485 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T23:36:18,485 WARN [BP-330010681-172.17.0.2-1731972917172 heartbeating to localhost/127.0.0.1:37995 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-330010681-172.17.0.2-1731972917172 (Datanode Uuid 1706733d-e884-4ee7-9e2f-89cc9dfd17a3) service to localhost/127.0.0.1:37995 2024-11-18T23:36:18,485 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T23:36:18,486 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data3/current/BP-330010681-172.17.0.2-1731972917172 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T23:36:18,486 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T23:36:18,487 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data4/current/BP-330010681-172.17.0.2-1731972917172 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T23:36:18,490 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@516a39dc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T23:36:18,491 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@428881d6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T23:36:18,491 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T23:36:18,491 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c68adb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T23:36:18,491 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@302ee775{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/hadoop.log.dir/,STOPPED} 2024-11-18T23:36:18,493 WARN [BP-330010681-172.17.0.2-1731972917172 heartbeating to localhost/127.0.0.1:37995 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T23:36:18,494 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T23:36:18,494 WARN [BP-330010681-172.17.0.2-1731972917172 heartbeating to localhost/127.0.0.1:37995 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-330010681-172.17.0.2-1731972917172 (Datanode Uuid e6f83191-0b6b-44b4-89ac-e00870a62215) service to localhost/127.0.0.1:37995 2024-11-18T23:36:18,494 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T23:36:18,495 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data2/current/BP-330010681-172.17.0.2-1731972917172 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T23:36:18,496 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T23:36:18,496 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/cluster_658e8dd9-1da4-2279-e865-a790ad3f2fb6/data/data1/current/BP-330010681-172.17.0.2-1731972917172 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T23:36:18,510 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7fe9bfd3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T23:36:18,511 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16695a38{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T23:36:18,512 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T23:36:18,512 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d655053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T23:36:18,512 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b5fa2d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b2339d50-fec6-398f-07ad-78512699ccfa/hadoop.log.dir/,STOPPED} 2024-11-18T23:36:18,528 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T23:36:18,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down