2024-11-14 09:29:08,282 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-14 09:29:08,296 main DEBUG Took 0.012297 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 09:29:08,297 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 09:29:08,297 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 09:29:08,298 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 09:29:08,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,307 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 09:29:08,319 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,321 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,321 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,322 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,322 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,322 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,323 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,324 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,325 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,326 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,326 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,327 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,327 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,327 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,327 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,328 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,328 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,329 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,329 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,329 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,330 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:29:08,330 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,330 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 09:29:08,332 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:29:08,333 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 09:29:08,335 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 09:29:08,335 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 09:29:08,336 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 09:29:08,337 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 09:29:08,345 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 09:29:08,347 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 09:29:08,349 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 09:29:08,349 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 09:29:08,349 main DEBUG createAppenders(={Console}) 2024-11-14 09:29:08,350 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-14 09:29:08,350 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-14 09:29:08,351 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-14 09:29:08,351 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 09:29:08,351 main DEBUG OutputStream closed 2024-11-14 09:29:08,352 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 09:29:08,352 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 09:29:08,352 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-14 09:29:08,427 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 09:29:08,430 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 09:29:08,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 09:29:08,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 09:29:08,434 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 09:29:08,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 09:29:08,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 09:29:08,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 09:29:08,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 09:29:08,437 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 09:29:08,437 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 09:29:08,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 09:29:08,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 09:29:08,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 09:29:08,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 09:29:08,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 09:29:08,440 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 09:29:08,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 09:29:08,443 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 09:29:08,444 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-14 09:29:08,444 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 09:29:08,445 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-14T09:29:08,718 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d 2024-11-14 09:29:08,721 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 09:29:08,722 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T09:29:08,732 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-14T09:29:08,762 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T09:29:08,809 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T09:29:08,810 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T09:29:08,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T09:29:08,842 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152, deleteOnExit=true 2024-11-14T09:29:08,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T09:29:08,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/test.cache.data in system properties and HBase conf 2024-11-14T09:29:08,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T09:29:08,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/hadoop.log.dir in system properties and HBase conf 2024-11-14T09:29:08,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T09:29:08,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T09:29:08,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T09:29:08,937 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T09:29:09,053 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:29:09,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T09:29:09,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T09:29:09,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T09:29:09,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T09:29:09,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T09:29:09,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T09:29:09,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T09:29:09,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T09:29:09,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T09:29:09,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/nfs.dump.dir in system properties and HBase conf 2024-11-14T09:29:09,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/java.io.tmpdir in system properties and HBase conf 2024-11-14T09:29:09,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T09:29:09,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T09:29:09,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T09:29:09,913 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T09:29:09,994 INFO [Time-limited test {}] log.Log(170): Logging initialized @2512ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T09:29:10,082 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:29:10,168 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T09:29:10,192 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T09:29:10,193 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T09:29:10,194 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T09:29:10,206 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:29:10,209 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39b35319{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/hadoop.log.dir/,AVAILABLE} 2024-11-14T09:29:10,210 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@377439aa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T09:29:10,401 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@25dc94e0{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/java.io.tmpdir/jetty-localhost-45749-hadoop-hdfs-3_4_1-tests_jar-_-any-15361691788081109077/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T09:29:10,407 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7cc108fc{HTTP/1.1, (http/1.1)}{localhost:45749} 2024-11-14T09:29:10,408 INFO [Time-limited test {}] server.Server(415): Started @2926ms 2024-11-14T09:29:10,815 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:29:10,824 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T09:29:10,825 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T09:29:10,826 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T09:29:10,826 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T09:29:10,827 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@66bf9098{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/hadoop.log.dir/,AVAILABLE} 2024-11-14T09:29:10,828 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8f47f6d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T09:29:10,963 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c8aad7c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/java.io.tmpdir/jetty-localhost-43281-hadoop-hdfs-3_4_1-tests_jar-_-any-13433452164367286097/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:29:10,964 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d06fb64{HTTP/1.1, (http/1.1)}{localhost:43281} 2024-11-14T09:29:10,964 INFO [Time-limited test {}] server.Server(415): Started @3482ms 2024-11-14T09:29:11,038 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T09:29:11,215 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:29:11,224 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T09:29:11,237 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T09:29:11,237 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T09:29:11,237 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T09:29:11,238 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5deb6e4f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/hadoop.log.dir/,AVAILABLE} 2024-11-14T09:29:11,239 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@363c876e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T09:29:11,398 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@23205bfa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/java.io.tmpdir/jetty-localhost-37483-hadoop-hdfs-3_4_1-tests_jar-_-any-3380642839652877409/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:29:11,399 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1df40b66{HTTP/1.1, (http/1.1)}{localhost:37483} 2024-11-14T09:29:11,399 INFO [Time-limited test {}] server.Server(415): Started @3917ms 2024-11-14T09:29:11,402 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T09:29:11,450 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:29:11,461 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T09:29:11,480 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T09:29:11,480 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T09:29:11,481 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T09:29:11,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1576ddbd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/hadoop.log.dir/,AVAILABLE} 2024-11-14T09:29:11,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@73d49bd7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T09:29:11,622 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@37ee96b7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/java.io.tmpdir/jetty-localhost-38429-hadoop-hdfs-3_4_1-tests_jar-_-any-14866912227954534910/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:29:11,633 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data1/current/BP-376496606-172.17.0.2-1731576549671/current, will proceed with Du for space computation calculation, 2024-11-14T09:29:11,634 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data2/current/BP-376496606-172.17.0.2-1731576549671/current, will proceed with Du for space computation calculation, 2024-11-14T09:29:11,634 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3b82d1a{HTTP/1.1, (http/1.1)}{localhost:38429} 2024-11-14T09:29:11,635 INFO [Time-limited test {}] server.Server(415): Started @4153ms 2024-11-14T09:29:11,638 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T09:29:11,640 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data3/current/BP-376496606-172.17.0.2-1731576549671/current, will proceed with Du for space computation calculation, 2024-11-14T09:29:11,643 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data4/current/BP-376496606-172.17.0.2-1731576549671/current, will proceed with Du for space computation calculation, 2024-11-14T09:29:11,767 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T09:29:11,771 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T09:29:11,899 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6630cab2f4b24c64 with lease ID 0x6340157334d5225f: Processing first storage report for DS-860268cd-ff3b-48ac-aa99-1273241384d4 from datanode DatanodeRegistration(127.0.0.1:41175, datanodeUuid=cd6cfbf2-5c89-44ac-8692-c27ea5b719ab, infoPort=40477, infoSecurePort=0, ipcPort=45337, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671) 2024-11-14T09:29:11,901 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6630cab2f4b24c64 with lease ID 0x6340157334d5225f: from storage DS-860268cd-ff3b-48ac-aa99-1273241384d4 node DatanodeRegistration(127.0.0.1:41175, datanodeUuid=cd6cfbf2-5c89-44ac-8692-c27ea5b719ab, infoPort=40477, infoSecurePort=0, ipcPort=45337, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-14T09:29:11,902 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8bbca1383d3b1e51 with lease ID 0x6340157334d5225e: Processing first storage report for DS-a51058c8-5b64-4eb0-9983-3592a1dfd434 from datanode DatanodeRegistration(127.0.0.1:43439, datanodeUuid=a74e73bc-bb0c-4473-aeb7-28477c3b69ea, infoPort=35591, infoSecurePort=0, ipcPort=44091, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671) 2024-11-14T09:29:11,902 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8bbca1383d3b1e51 with lease ID 0x6340157334d5225e: from storage DS-a51058c8-5b64-4eb0-9983-3592a1dfd434 node DatanodeRegistration(127.0.0.1:43439, datanodeUuid=a74e73bc-bb0c-4473-aeb7-28477c3b69ea, infoPort=35591, infoSecurePort=0, ipcPort=44091, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T09:29:11,903 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6630cab2f4b24c64 with lease ID 0x6340157334d5225f: Processing first storage report for DS-4561525f-2af7-4e15-9bf8-82f89d58e9a9 from datanode DatanodeRegistration(127.0.0.1:41175, datanodeUuid=cd6cfbf2-5c89-44ac-8692-c27ea5b719ab, infoPort=40477, infoSecurePort=0, ipcPort=45337, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671) 2024-11-14T09:29:11,903 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6630cab2f4b24c64 with lease ID 0x6340157334d5225f: from storage DS-4561525f-2af7-4e15-9bf8-82f89d58e9a9 node DatanodeRegistration(127.0.0.1:41175, datanodeUuid=cd6cfbf2-5c89-44ac-8692-c27ea5b719ab, infoPort=40477, infoSecurePort=0, ipcPort=45337, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T09:29:11,903 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8bbca1383d3b1e51 with lease ID 0x6340157334d5225e: Processing first storage report for DS-85dbcee0-ded0-4dcf-a824-6282c294f7d7 from datanode DatanodeRegistration(127.0.0.1:43439, datanodeUuid=a74e73bc-bb0c-4473-aeb7-28477c3b69ea, infoPort=35591, infoSecurePort=0, ipcPort=44091, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671) 2024-11-14T09:29:11,904 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8bbca1383d3b1e51 with lease ID 0x6340157334d5225e: from storage DS-85dbcee0-ded0-4dcf-a824-6282c294f7d7 node DatanodeRegistration(127.0.0.1:43439, datanodeUuid=a74e73bc-bb0c-4473-aeb7-28477c3b69ea, infoPort=35591, infoSecurePort=0, ipcPort=44091, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T09:29:11,936 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data5/current/BP-376496606-172.17.0.2-1731576549671/current, will proceed with Du for space computation calculation, 2024-11-14T09:29:11,936 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data6/current/BP-376496606-172.17.0.2-1731576549671/current, will proceed with Du for space computation calculation, 2024-11-14T09:29:11,981 WARN [Thread-123 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T09:29:11,986 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd426be0e9542ef8a with lease ID 0x6340157334d52260: Processing first storage report for DS-5e08a043-ecd3-4be0-878e-2a81fea495ce from datanode DatanodeRegistration(127.0.0.1:37113, datanodeUuid=4f0a233f-d3b7-47c2-ae66-d7e05aff4523, infoPort=39845, infoSecurePort=0, ipcPort=45747, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671) 2024-11-14T09:29:11,987 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd426be0e9542ef8a with lease ID 0x6340157334d52260: from storage DS-5e08a043-ecd3-4be0-878e-2a81fea495ce node DatanodeRegistration(127.0.0.1:37113, datanodeUuid=4f0a233f-d3b7-47c2-ae66-d7e05aff4523, infoPort=39845, infoSecurePort=0, ipcPort=45747, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T09:29:11,987 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd426be0e9542ef8a with lease ID 0x6340157334d52260: Processing first storage report for DS-73f238ea-7705-4aac-b08d-ba8108369117 from datanode DatanodeRegistration(127.0.0.1:37113, datanodeUuid=4f0a233f-d3b7-47c2-ae66-d7e05aff4523, infoPort=39845, infoSecurePort=0, ipcPort=45747, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671) 2024-11-14T09:29:11,987 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd426be0e9542ef8a with lease ID 0x6340157334d52260: from storage DS-73f238ea-7705-4aac-b08d-ba8108369117 node DatanodeRegistration(127.0.0.1:37113, datanodeUuid=4f0a233f-d3b7-47c2-ae66-d7e05aff4523, infoPort=39845, infoSecurePort=0, ipcPort=45747, storageInfo=lv=-57;cid=testClusterID;nsid=1917621442;c=1731576549671), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T09:29:12,206 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d 2024-11-14T09:29:12,316 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/zookeeper_0, clientPort=59004, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T09:29:12,327 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59004 2024-11-14T09:29:12,341 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:12,343 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:12,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741825_1001 (size=7) 2024-11-14T09:29:12,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741825_1001 (size=7) 2024-11-14T09:29:12,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741825_1001 (size=7) 2024-11-14T09:29:12,975 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963 with version=8 2024-11-14T09:29:12,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/hbase-staging 2024-11-14T09:29:13,321 INFO [Time-limited test {}] client.ConnectionUtils(128): master/83f56b55f2af:0 server-side Connection retries=45 2024-11-14T09:29:13,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:13,333 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:13,338 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:29:13,338 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:13,338 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:29:13,506 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T09:29:13,586 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T09:29:13,597 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T09:29:13,603 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:29:13,634 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 24698 (auto-detected) 2024-11-14T09:29:13,635 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T09:29:13,657 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43303 2024-11-14T09:29:13,677 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43303 connecting to ZooKeeper ensemble=127.0.0.1:59004 2024-11-14T09:29:13,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433030x0, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:29:13,717 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43303-0x10115cf45f40000 connected 2024-11-14T09:29:13,750 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:13,754 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:13,765 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:29:13,770 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963, hbase.cluster.distributed=false 2024-11-14T09:29:13,792 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:29:13,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43303 2024-11-14T09:29:13,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43303 2024-11-14T09:29:13,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43303 2024-11-14T09:29:13,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43303 2024-11-14T09:29:13,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43303 2024-11-14T09:29:13,921 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/83f56b55f2af:0 server-side Connection retries=45 2024-11-14T09:29:13,923 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:13,924 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:13,924 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:29:13,924 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:13,924 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:29:13,927 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T09:29:13,930 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:29:13,931 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32943 2024-11-14T09:29:13,933 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32943 connecting to ZooKeeper ensemble=127.0.0.1:59004 2024-11-14T09:29:13,934 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:13,936 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:13,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:329430x0, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:29:13,943 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:329430x0, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:29:13,943 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32943-0x10115cf45f40001 connected 2024-11-14T09:29:13,947 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T09:29:13,956 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T09:29:13,958 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T09:29:13,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:29:13,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32943 2024-11-14T09:29:13,965 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32943 2024-11-14T09:29:13,965 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32943 2024-11-14T09:29:13,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32943 2024-11-14T09:29:13,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32943 2024-11-14T09:29:13,984 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/83f56b55f2af:0 server-side Connection retries=45 2024-11-14T09:29:13,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:13,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:13,985 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:29:13,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:13,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:29:13,986 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T09:29:13,986 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:29:13,987 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43391 2024-11-14T09:29:13,989 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43391 connecting to ZooKeeper ensemble=127.0.0.1:59004 2024-11-14T09:29:13,990 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:13,992 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:13,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:433910x0, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:29:13,997 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43391-0x10115cf45f40002 connected 2024-11-14T09:29:13,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:29:13,998 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T09:29:13,999 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T09:29:14,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T09:29:14,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:29:14,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43391 2024-11-14T09:29:14,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43391 2024-11-14T09:29:14,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43391 2024-11-14T09:29:14,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43391 2024-11-14T09:29:14,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43391 2024-11-14T09:29:14,024 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/83f56b55f2af:0 server-side Connection retries=45 2024-11-14T09:29:14,024 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:14,024 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:14,025 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:29:14,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:29:14,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:29:14,025 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T09:29:14,025 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:29:14,026 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35921 2024-11-14T09:29:14,027 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35921 connecting to ZooKeeper ensemble=127.0.0.1:59004 2024-11-14T09:29:14,029 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:14,030 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:14,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359210x0, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:29:14,036 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:359210x0, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:29:14,036 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35921-0x10115cf45f40003 connected 2024-11-14T09:29:14,036 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T09:29:14,040 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T09:29:14,041 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T09:29:14,043 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:29:14,044 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35921 2024-11-14T09:29:14,044 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35921 2024-11-14T09:29:14,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35921 2024-11-14T09:29:14,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35921 2024-11-14T09:29:14,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35921 2024-11-14T09:29:14,065 DEBUG [M:0;83f56b55f2af:43303 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;83f56b55f2af:43303 2024-11-14T09:29:14,066 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/83f56b55f2af,43303,1731576553096 2024-11-14T09:29:14,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:29:14,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:29:14,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:29:14,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:29:14,080 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/83f56b55f2af,43303,1731576553096 2024-11-14T09:29:14,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T09:29:14,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T09:29:14,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T09:29:14,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,108 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T09:29:14,110 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/83f56b55f2af,43303,1731576553096 from backup master directory 2024-11-14T09:29:14,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/83f56b55f2af,43303,1731576553096 2024-11-14T09:29:14,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:29:14,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:29:14,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:29:14,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:29:14,114 WARN [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:29:14,114 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=83f56b55f2af,43303,1731576553096 2024-11-14T09:29:14,117 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T09:29:14,118 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T09:29:14,177 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/hbase.id] with ID: b68c35d9-ffc1-44fd-aa0b-01bef063b027 2024-11-14T09:29:14,177 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/.tmp/hbase.id 2024-11-14T09:29:14,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741826_1002 (size=42) 2024-11-14T09:29:14,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741826_1002 (size=42) 2024-11-14T09:29:14,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741826_1002 (size=42) 2024-11-14T09:29:14,193 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/.tmp/hbase.id]:[hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/hbase.id] 2024-11-14T09:29:14,241 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:29:14,248 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T09:29:14,274 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-14T09:29:14,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741827_1003 (size=196) 2024-11-14T09:29:14,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741827_1003 (size=196) 2024-11-14T09:29:14,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741827_1003 (size=196) 2024-11-14T09:29:14,318 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:29:14,320 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T09:29:14,327 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:14,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741828_1004 (size=1189) 2024-11-14T09:29:14,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741828_1004 (size=1189) 2024-11-14T09:29:14,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741828_1004 (size=1189) 2024-11-14T09:29:14,384 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store 2024-11-14T09:29:14,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741829_1005 (size=34) 2024-11-14T09:29:14,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741829_1005 (size=34) 2024-11-14T09:29:14,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741829_1005 (size=34) 2024-11-14T09:29:14,419 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T09:29:14,424 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:14,425 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T09:29:14,425 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:29:14,426 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:29:14,428 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T09:29:14,428 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:29:14,428 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:29:14,430 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731576554425Disabling compacts and flushes for region at 1731576554425Disabling writes for close at 1731576554428 (+3 ms)Writing region close event to WAL at 1731576554428Closed at 1731576554428 2024-11-14T09:29:14,432 WARN [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/.initializing 2024-11-14T09:29:14,432 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/WALs/83f56b55f2af,43303,1731576553096 2024-11-14T09:29:14,441 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:14,456 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83f56b55f2af%2C43303%2C1731576553096, suffix=, logDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/WALs/83f56b55f2af,43303,1731576553096, archiveDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/oldWALs, maxLogs=10 2024-11-14T09:29:14,486 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/WALs/83f56b55f2af,43303,1731576553096/83f56b55f2af%2C43303%2C1731576553096.1731576554460, exclude list is [], retry=0 2024-11-14T09:29:14,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:14,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:14,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:14,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:14,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T09:29:14,557 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/WALs/83f56b55f2af,43303,1731576553096/83f56b55f2af%2C43303%2C1731576553096.1731576554460 2024-11-14T09:29:14,558 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:14,559 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:14,559 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:14,562 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,563 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T09:29:14,629 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:14,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:14,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,635 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T09:29:14,635 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:14,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:14,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T09:29:14,639 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:14,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:14,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T09:29:14,644 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:14,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:14,645 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,649 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,650 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,655 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,656 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,659 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T09:29:14,663 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:29:14,668 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:14,670 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59201063, jitterRate=-0.11783541738986969}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T09:29:14,678 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731576554576Initializing all the Stores at 1731576554578 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576554578Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576554579 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576554579Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576554579Cleaning up temporary data from old regions at 1731576554656 (+77 ms)Region opened successfully at 1731576554678 (+22 ms) 2024-11-14T09:29:14,680 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T09:29:14,717 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ccd0f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=83f56b55f2af/172.17.0.2:0 2024-11-14T09:29:14,750 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T09:29:14,761 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T09:29:14,762 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T09:29:14,765 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T09:29:14,766 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T09:29:14,771 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T09:29:14,771 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T09:29:14,797 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T09:29:14,808 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T09:29:14,811 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T09:29:14,814 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T09:29:14,816 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T09:29:14,817 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T09:29:14,820 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T09:29:14,824 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T09:29:14,826 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T09:29:14,827 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T09:29:14,829 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T09:29:14,849 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T09:29:14,851 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T09:29:14,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:29:14,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:29:14,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:29:14,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:29:14,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,861 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=83f56b55f2af,43303,1731576553096, sessionid=0x10115cf45f40000, setting cluster-up flag (Was=false) 2024-11-14T09:29:14,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,882 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T09:29:14,883 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=83f56b55f2af,43303,1731576553096 2024-11-14T09:29:14,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:14,896 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T09:29:14,897 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=83f56b55f2af,43303,1731576553096 2024-11-14T09:29:14,904 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T09:29:14,954 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(746): ClusterId : b68c35d9-ffc1-44fd-aa0b-01bef063b027 2024-11-14T09:29:14,954 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(746): ClusterId : b68c35d9-ffc1-44fd-aa0b-01bef063b027 2024-11-14T09:29:14,954 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(746): ClusterId : b68c35d9-ffc1-44fd-aa0b-01bef063b027 2024-11-14T09:29:14,957 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T09:29:14,957 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T09:29:14,957 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T09:29:14,966 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T09:29:14,966 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T09:29:14,966 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T09:29:14,966 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T09:29:14,969 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T09:29:14,969 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T09:29:14,970 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T09:29:14,970 DEBUG [RS:0;83f56b55f2af:32943 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110783a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=83f56b55f2af/172.17.0.2:0 2024-11-14T09:29:14,977 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T09:29:14,978 DEBUG [RS:2;83f56b55f2af:35921 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7974dbbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=83f56b55f2af/172.17.0.2:0 2024-11-14T09:29:14,979 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T09:29:14,980 DEBUG [RS:1;83f56b55f2af:43391 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5d858e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=83f56b55f2af/172.17.0.2:0 2024-11-14T09:29:14,988 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T09:29:14,991 DEBUG [RS:0;83f56b55f2af:32943 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;83f56b55f2af:32943 2024-11-14T09:29:14,994 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T09:29:14,995 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T09:29:14,995 DEBUG [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T09:29:14,998 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(2659): reportForDuty to master=83f56b55f2af,43303,1731576553096 with port=32943, startcode=1731576553877 2024-11-14T09:29:15,001 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T09:29:15,001 DEBUG [RS:2;83f56b55f2af:35921 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;83f56b55f2af:35921 2024-11-14T09:29:15,001 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T09:29:15,001 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T09:29:15,001 DEBUG [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T09:29:15,003 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(2659): reportForDuty to master=83f56b55f2af,43303,1731576553096 with port=35921, startcode=1731576554024 2024-11-14T09:29:15,009 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T09:29:15,012 DEBUG [RS:0;83f56b55f2af:32943 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T09:29:15,012 DEBUG [RS:2;83f56b55f2af:35921 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T09:29:15,016 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 83f56b55f2af,43303,1731576553096 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T09:29:15,021 DEBUG [RS:1;83f56b55f2af:43391 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;83f56b55f2af:43391 2024-11-14T09:29:15,021 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T09:29:15,021 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T09:29:15,021 DEBUG [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T09:29:15,023 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(2659): reportForDuty to master=83f56b55f2af,43303,1731576553096 with port=43391, startcode=1731576553984 2024-11-14T09:29:15,023 DEBUG [RS:1;83f56b55f2af:43391 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T09:29:15,025 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:29:15,025 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:29:15,026 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:29:15,026 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/83f56b55f2af:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:29:15,026 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/83f56b55f2af:0, corePoolSize=10, maxPoolSize=10 2024-11-14T09:29:15,026 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,026 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/83f56b55f2af:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:29:15,027 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,041 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731576585041 2024-11-14T09:29:15,043 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T09:29:15,045 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T09:29:15,050 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T09:29:15,051 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T09:29:15,051 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T09:29:15,051 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T09:29:15,053 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:29:15,054 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T09:29:15,063 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38109, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T09:29:15,060 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,064 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41547, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T09:29:15,065 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:15,065 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T09:29:15,066 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T09:29:15,067 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55351, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T09:29:15,068 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T09:29:15,068 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T09:29:15,071 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T09:29:15,072 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T09:29:15,072 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43303 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:29:15,077 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.large.0-1731576555074,5,FailOnTimeoutGroup] 2024-11-14T09:29:15,078 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.small.0-1731576555077,5,FailOnTimeoutGroup] 2024-11-14T09:29:15,079 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,079 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T09:29:15,079 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43303 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:29:15,080 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,080 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43303 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T09:29:15,081 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741831_1007 (size=1321) 2024-11-14T09:29:15,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741831_1007 (size=1321) 2024-11-14T09:29:15,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741831_1007 (size=1321) 2024-11-14T09:29:15,136 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T09:29:15,137 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963 2024-11-14T09:29:15,146 DEBUG [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T09:29:15,146 DEBUG [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T09:29:15,146 DEBUG [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T09:29:15,146 WARN [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T09:29:15,146 WARN [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T09:29:15,146 WARN [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T09:29:15,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741832_1008 (size=32) 2024-11-14T09:29:15,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741832_1008 (size=32) 2024-11-14T09:29:15,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741832_1008 (size=32) 2024-11-14T09:29:15,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:15,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T09:29:15,159 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:29:15,159 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:15,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:15,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T09:29:15,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:29:15,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:15,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:15,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T09:29:15,166 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:29:15,166 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:15,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:15,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T09:29:15,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:29:15,170 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:15,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:15,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:29:15,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740 2024-11-14T09:29:15,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740 2024-11-14T09:29:15,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:29:15,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:29:15,177 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T09:29:15,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:29:15,184 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:15,185 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66366823, jitterRate=-0.011057272553443909}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T09:29:15,187 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731576555154Initializing all the Stores at 1731576555156 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576555157 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576555157Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576555157Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576555157Cleaning up temporary data from old regions at 1731576555176 (+19 ms)Region opened successfully at 1731576555187 (+11 ms) 2024-11-14T09:29:15,187 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:29:15,188 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:29:15,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:29:15,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:29:15,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:29:15,189 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:29:15,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731576555187Disabling compacts and flushes for region at 1731576555187Disabling writes for close at 1731576555188 (+1 ms)Writing region close event to WAL at 1731576555189 (+1 ms)Closed at 1731576555189 2024-11-14T09:29:15,193 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:29:15,193 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T09:29:15,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T09:29:15,208 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:29:15,212 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T09:29:15,248 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(2659): reportForDuty to master=83f56b55f2af,43303,1731576553096 with port=35921, startcode=1731576554024 2024-11-14T09:29:15,248 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(2659): reportForDuty to master=83f56b55f2af,43303,1731576553096 with port=32943, startcode=1731576553877 2024-11-14T09:29:15,248 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(2659): reportForDuty to master=83f56b55f2af,43303,1731576553096 with port=43391, startcode=1731576553984 2024-11-14T09:29:15,250 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43303 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 83f56b55f2af,43391,1731576553984 2024-11-14T09:29:15,253 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43303 {}] master.ServerManager(517): Registering regionserver=83f56b55f2af,43391,1731576553984 2024-11-14T09:29:15,261 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43303 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 83f56b55f2af,35921,1731576554024 2024-11-14T09:29:15,261 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43303 {}] master.ServerManager(517): Registering regionserver=83f56b55f2af,35921,1731576554024 2024-11-14T09:29:15,262 DEBUG [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963 2024-11-14T09:29:15,262 DEBUG [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34615 2024-11-14T09:29:15,262 DEBUG [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T09:29:15,264 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43303 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 83f56b55f2af,32943,1731576553877 2024-11-14T09:29:15,264 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43303 {}] master.ServerManager(517): Registering regionserver=83f56b55f2af,32943,1731576553877 2024-11-14T09:29:15,264 DEBUG [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963 2024-11-14T09:29:15,264 DEBUG [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34615 2024-11-14T09:29:15,264 DEBUG [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T09:29:15,267 DEBUG [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963 2024-11-14T09:29:15,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:29:15,267 DEBUG [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34615 2024-11-14T09:29:15,267 DEBUG [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T09:29:15,270 DEBUG [RS:1;83f56b55f2af:43391 {}] zookeeper.ZKUtil(111): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/83f56b55f2af,43391,1731576553984 2024-11-14T09:29:15,270 WARN [RS:1;83f56b55f2af:43391 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:29:15,270 INFO [RS:1;83f56b55f2af:43391 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:15,271 DEBUG [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,43391,1731576553984 2024-11-14T09:29:15,271 DEBUG [RS:2;83f56b55f2af:35921 {}] zookeeper.ZKUtil(111): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/83f56b55f2af,35921,1731576554024 2024-11-14T09:29:15,271 WARN [RS:2;83f56b55f2af:35921 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:29:15,272 INFO [RS:2;83f56b55f2af:35921 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:15,272 DEBUG [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024 2024-11-14T09:29:15,273 DEBUG [RS:0;83f56b55f2af:32943 {}] zookeeper.ZKUtil(111): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/83f56b55f2af,32943,1731576553877 2024-11-14T09:29:15,273 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [83f56b55f2af,35921,1731576554024] 2024-11-14T09:29:15,273 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [83f56b55f2af,43391,1731576553984] 2024-11-14T09:29:15,273 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [83f56b55f2af,32943,1731576553877] 2024-11-14T09:29:15,273 WARN [RS:0;83f56b55f2af:32943 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:29:15,274 INFO [RS:0;83f56b55f2af:32943 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:15,274 DEBUG [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,32943,1731576553877 2024-11-14T09:29:15,303 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T09:29:15,303 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T09:29:15,303 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T09:29:15,317 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T09:29:15,317 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T09:29:15,318 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T09:29:15,323 INFO [RS:0;83f56b55f2af:32943 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T09:29:15,323 INFO [RS:1;83f56b55f2af:43391 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T09:29:15,323 INFO [RS:2;83f56b55f2af:35921 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T09:29:15,323 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,323 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,323 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,324 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T09:29:15,324 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T09:29:15,324 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T09:29:15,331 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T09:29:15,331 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T09:29:15,331 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T09:29:15,332 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,332 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,332 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,333 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:29:15,333 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,333 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:29:15,334 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:29:15,334 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/83f56b55f2af:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:29:15,334 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:2;83f56b55f2af:35921 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:29:15,334 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,334 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,335 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,335 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,335 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/83f56b55f2af:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:29:15,335 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/83f56b55f2af:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:29:15,335 DEBUG [RS:1;83f56b55f2af:43391 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:29:15,335 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/83f56b55f2af:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:29:15,335 DEBUG [RS:0;83f56b55f2af:32943 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/83f56b55f2af:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:29:15,346 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,346 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,346 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,346 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,346 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,346 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,347 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,347 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,43391,1731576553984-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:29:15,347 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,347 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,347 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,347 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,35921,1731576554024-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:29:15,348 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,348 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,348 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,348 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,348 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,348 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,32943,1731576553877-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:29:15,363 WARN [83f56b55f2af:43303 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T09:29:15,375 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T09:29:15,377 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T09:29:15,377 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,43391,1731576553984-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,378 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,32943,1731576553877-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,378 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,378 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,378 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.Replication(171): 83f56b55f2af,32943,1731576553877 started 2024-11-14T09:29:15,378 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.Replication(171): 83f56b55f2af,43391,1731576553984 started 2024-11-14T09:29:15,378 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T09:29:15,379 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,35921,1731576554024-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,379 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,379 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.Replication(171): 83f56b55f2af,35921,1731576554024 started 2024-11-14T09:29:15,398 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,398 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(1482): Serving as 83f56b55f2af,32943,1731576553877, RpcServer on 83f56b55f2af/172.17.0.2:32943, sessionid=0x10115cf45f40001 2024-11-14T09:29:15,399 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T09:29:15,399 DEBUG [RS:0;83f56b55f2af:32943 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 83f56b55f2af,32943,1731576553877 2024-11-14T09:29:15,399 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83f56b55f2af,32943,1731576553877' 2024-11-14T09:29:15,400 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T09:29:15,401 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T09:29:15,401 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T09:29:15,402 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T09:29:15,402 DEBUG [RS:0;83f56b55f2af:32943 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 83f56b55f2af,32943,1731576553877 2024-11-14T09:29:15,402 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83f56b55f2af,32943,1731576553877' 2024-11-14T09:29:15,402 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T09:29:15,403 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T09:29:15,403 DEBUG [RS:0;83f56b55f2af:32943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T09:29:15,403 INFO [RS:0;83f56b55f2af:32943 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T09:29:15,403 INFO [RS:0;83f56b55f2af:32943 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T09:29:15,404 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,404 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:15,404 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(1482): Serving as 83f56b55f2af,43391,1731576553984, RpcServer on 83f56b55f2af/172.17.0.2:43391, sessionid=0x10115cf45f40002 2024-11-14T09:29:15,404 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(1482): Serving as 83f56b55f2af,35921,1731576554024, RpcServer on 83f56b55f2af/172.17.0.2:35921, sessionid=0x10115cf45f40003 2024-11-14T09:29:15,404 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T09:29:15,404 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T09:29:15,404 DEBUG [RS:2;83f56b55f2af:35921 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 83f56b55f2af,35921,1731576554024 2024-11-14T09:29:15,404 DEBUG [RS:1;83f56b55f2af:43391 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 83f56b55f2af,43391,1731576553984 2024-11-14T09:29:15,404 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83f56b55f2af,35921,1731576554024' 2024-11-14T09:29:15,404 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83f56b55f2af,43391,1731576553984' 2024-11-14T09:29:15,404 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T09:29:15,404 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T09:29:15,405 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T09:29:15,405 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T09:29:15,406 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T09:29:15,406 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T09:29:15,406 DEBUG [RS:1;83f56b55f2af:43391 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 83f56b55f2af,43391,1731576553984 2024-11-14T09:29:15,406 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T09:29:15,406 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83f56b55f2af,43391,1731576553984' 2024-11-14T09:29:15,406 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T09:29:15,406 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T09:29:15,406 DEBUG [RS:2;83f56b55f2af:35921 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 83f56b55f2af,35921,1731576554024 2024-11-14T09:29:15,406 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '83f56b55f2af,35921,1731576554024' 2024-11-14T09:29:15,406 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T09:29:15,407 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T09:29:15,407 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T09:29:15,407 DEBUG [RS:1;83f56b55f2af:43391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T09:29:15,407 INFO [RS:1;83f56b55f2af:43391 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T09:29:15,407 INFO [RS:1;83f56b55f2af:43391 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T09:29:15,408 DEBUG [RS:2;83f56b55f2af:35921 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T09:29:15,408 INFO [RS:2;83f56b55f2af:35921 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T09:29:15,408 INFO [RS:2;83f56b55f2af:35921 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T09:29:15,509 INFO [RS:2;83f56b55f2af:35921 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:15,509 INFO [RS:0;83f56b55f2af:32943 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:15,509 INFO [RS:1;83f56b55f2af:43391 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:15,513 INFO [RS:0;83f56b55f2af:32943 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83f56b55f2af%2C32943%2C1731576553877, suffix=, logDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,32943,1731576553877, archiveDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/oldWALs, maxLogs=32 2024-11-14T09:29:15,513 INFO [RS:2;83f56b55f2af:35921 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83f56b55f2af%2C35921%2C1731576554024, suffix=, logDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024, archiveDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/oldWALs, maxLogs=32 2024-11-14T09:29:15,514 INFO [RS:1;83f56b55f2af:43391 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83f56b55f2af%2C43391%2C1731576553984, suffix=, logDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,43391,1731576553984, archiveDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/oldWALs, maxLogs=32 2024-11-14T09:29:15,537 DEBUG [RS:1;83f56b55f2af:43391 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,43391,1731576553984/83f56b55f2af%2C43391%2C1731576553984.1731576555518, exclude list is [], retry=0 2024-11-14T09:29:15,537 DEBUG [RS:2;83f56b55f2af:35921 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024/83f56b55f2af%2C35921%2C1731576554024.1731576555517, exclude list is [], retry=0 2024-11-14T09:29:15,537 DEBUG [RS:0;83f56b55f2af:32943 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,32943,1731576553877/83f56b55f2af%2C32943%2C1731576553877.1731576555518, exclude list is [], retry=0 2024-11-14T09:29:15,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:15,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:15,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:15,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:15,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:15,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:15,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:15,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:15,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:15,556 INFO [RS:1;83f56b55f2af:43391 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,43391,1731576553984/83f56b55f2af%2C43391%2C1731576553984.1731576555518 2024-11-14T09:29:15,557 INFO [RS:2;83f56b55f2af:35921 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024/83f56b55f2af%2C35921%2C1731576554024.1731576555517 2024-11-14T09:29:15,558 DEBUG [RS:1;83f56b55f2af:43391 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:15,558 DEBUG [RS:2;83f56b55f2af:35921 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:15,561 INFO [RS:0;83f56b55f2af:32943 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,32943,1731576553877/83f56b55f2af%2C32943%2C1731576553877.1731576555518 2024-11-14T09:29:15,561 DEBUG [RS:0;83f56b55f2af:32943 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:15,616 DEBUG [83f56b55f2af:43303 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-14T09:29:15,625 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(204): Hosts are {83f56b55f2af=0} racks are {/default-rack=0} 2024-11-14T09:29:15,632 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T09:29:15,632 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T09:29:15,632 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T09:29:15,632 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T09:29:15,632 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T09:29:15,632 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T09:29:15,632 INFO [83f56b55f2af:43303 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T09:29:15,632 INFO [83f56b55f2af:43303 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T09:29:15,632 INFO [83f56b55f2af:43303 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T09:29:15,632 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T09:29:15,639 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=83f56b55f2af,35921,1731576554024 2024-11-14T09:29:15,647 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 83f56b55f2af,35921,1731576554024, state=OPENING 2024-11-14T09:29:15,653 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T09:29:15,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:15,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:15,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:15,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:15,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:15,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:15,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:15,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:15,660 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:29:15,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=83f56b55f2af,35921,1731576554024}] 2024-11-14T09:29:15,844 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T09:29:15,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53943, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T09:29:15,860 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T09:29:15,860 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:15,861 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T09:29:15,864 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83f56b55f2af%2C35921%2C1731576554024.meta, suffix=.meta, logDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024, archiveDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/oldWALs, maxLogs=32 2024-11-14T09:29:15,888 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta, exclude list is [], retry=0 2024-11-14T09:29:15,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:15,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:15,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:15,897 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta 2024-11-14T09:29:15,898 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:15,898 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:15,900 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T09:29:15,904 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T09:29:15,911 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T09:29:15,916 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T09:29:15,917 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:15,917 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T09:29:15,917 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T09:29:15,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T09:29:15,923 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:29:15,923 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:15,924 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:15,924 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T09:29:15,925 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:29:15,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:15,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:15,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T09:29:15,928 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:29:15,928 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:15,929 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:15,929 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T09:29:15,930 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:29:15,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:15,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:15,931 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:29:15,933 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740 2024-11-14T09:29:15,936 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740 2024-11-14T09:29:15,939 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:29:15,939 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:29:15,940 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T09:29:15,943 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:29:15,944 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70865548, jitterRate=0.05597895383834839}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T09:29:15,945 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T09:29:15,946 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731576555918Writing region info on filesystem at 1731576555918Initializing all the Stores at 1731576555921 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576555921Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576555921Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576555921Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576555921Cleaning up temporary data from old regions at 1731576555939 (+18 ms)Running coprocessor post-open hooks at 1731576555945 (+6 ms)Region opened successfully at 1731576555946 (+1 ms) 2024-11-14T09:29:15,956 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731576555835 2024-11-14T09:29:15,970 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T09:29:15,970 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T09:29:15,972 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,35921,1731576554024 2024-11-14T09:29:15,974 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 83f56b55f2af,35921,1731576554024, state=OPEN 2024-11-14T09:29:15,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:29:15,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:29:15,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:29:15,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:29:15,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:15,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:15,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:15,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:15,978 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=83f56b55f2af,35921,1731576554024 2024-11-14T09:29:15,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T09:29:15,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=83f56b55f2af,35921,1731576554024 in 316 msec 2024-11-14T09:29:15,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T09:29:15,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 788 msec 2024-11-14T09:29:15,997 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:29:15,997 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T09:29:16,051 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:29:16,053 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1] 2024-11-14T09:29:16,087 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:29:16,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38867, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:29:16,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1790 sec 2024-11-14T09:29:16,114 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731576556114, completionTime=-1 2024-11-14T09:29:16,118 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-14T09:29:16,118 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T09:29:16,149 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-14T09:29:16,149 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731576616149 2024-11-14T09:29:16,149 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731576676149 2024-11-14T09:29:16,149 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-14T09:29:16,151 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-14T09:29:16,159 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,43303,1731576553096-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:16,159 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,43303,1731576553096-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:16,159 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,43303,1731576553096-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:16,161 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-83f56b55f2af:43303, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:16,161 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:16,164 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:16,169 DEBUG [master/83f56b55f2af:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T09:29:16,197 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.083sec 2024-11-14T09:29:16,199 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T09:29:16,201 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T09:29:16,202 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T09:29:16,203 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T09:29:16,203 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T09:29:16,204 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,43303,1731576553096-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:29:16,204 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,43303,1731576553096-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T09:29:16,217 DEBUG [master/83f56b55f2af:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T09:29:16,218 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T09:29:16,218 INFO [master/83f56b55f2af:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=83f56b55f2af,43303,1731576553096-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:16,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c24fbc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:29:16,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 83f56b55f2af,43303,-1 for getting cluster id 2024-11-14T09:29:16,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:29:16,278 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b68c35d9-ffc1-44fd-aa0b-01bef063b027' 2024-11-14T09:29:16,280 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:29:16,280 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b68c35d9-ffc1-44fd-aa0b-01bef063b027" 2024-11-14T09:29:16,281 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f56dc82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:29:16,281 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [83f56b55f2af,43303,-1] 2024-11-14T09:29:16,283 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:29:16,285 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:29:16,287 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:29:16,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20f7b9b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:29:16,291 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:29:16,300 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1] 2024-11-14T09:29:16,301 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:29:16,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:29:16,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=83f56b55f2af,43303,1731576553096 2024-11-14T09:29:16,331 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:34615/hbase 2024-11-14T09:29:16,351 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=361, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=7082 2024-11-14T09:29:16,371 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:16,375 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:16,376 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:16,381 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-78864879, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/hregion-78864879, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:16,406 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-78864879/hregion-78864879.1731576556383, exclude list is [], retry=0 2024-11-14T09:29:16,416 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:16,419 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:16,421 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:16,441 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-78864879/hregion-78864879.1731576556383 2024-11-14T09:29:16,445 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:16,445 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 63e0f39980e2ffcdf2d4594ca884d7df, NAME => 'testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/hbase 2024-11-14T09:29:16,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741838_1014 (size=64) 2024-11-14T09:29:16,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741838_1014 (size=64) 2024-11-14T09:29:16,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741838_1014 (size=64) 2024-11-14T09:29:16,470 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:16,474 INFO [StoreOpener-63e0f39980e2ffcdf2d4594ca884d7df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,477 INFO [StoreOpener-63e0f39980e2ffcdf2d4594ca884d7df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63e0f39980e2ffcdf2d4594ca884d7df columnFamilyName a 2024-11-14T09:29:16,478 DEBUG [StoreOpener-63e0f39980e2ffcdf2d4594ca884d7df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:16,478 INFO [StoreOpener-63e0f39980e2ffcdf2d4594ca884d7df-1 {}] regionserver.HStore(327): Store=63e0f39980e2ffcdf2d4594ca884d7df/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:16,479 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,480 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,481 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,482 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,482 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,485 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,489 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:16,490 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 63e0f39980e2ffcdf2d4594ca884d7df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66974328, jitterRate=-0.0020047426223754883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T09:29:16,491 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 63e0f39980e2ffcdf2d4594ca884d7df: Writing region info on filesystem at 1731576556470Initializing all the Stores at 1731576556472 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576556472Cleaning up temporary data from old regions at 1731576556482 (+10 ms)Region opened successfully at 1731576556491 (+9 ms) 2024-11-14T09:29:16,491 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 63e0f39980e2ffcdf2d4594ca884d7df, disabling compactions & flushes 2024-11-14T09:29:16,491 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df. 2024-11-14T09:29:16,492 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df. 2024-11-14T09:29:16,492 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df. after waiting 0 ms 2024-11-14T09:29:16,492 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df. 2024-11-14T09:29:16,492 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df. 2024-11-14T09:29:16,492 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 63e0f39980e2ffcdf2d4594ca884d7df: Waiting for close lock at 1731576556491Disabling compacts and flushes for region at 1731576556491Disabling writes for close at 1731576556492 (+1 ms)Writing region close event to WAL at 1731576556492Closed at 1731576556492 2024-11-14T09:29:16,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741837_1013 (size=93) 2024-11-14T09:29:16,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741837_1013 (size=93) 2024-11-14T09:29:16,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741837_1013 (size=93) 2024-11-14T09:29:16,519 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:16,519 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-78864879:(num 1731576556383) 2024-11-14T09:29:16,522 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T09:29:16,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741839_1015 (size=369) 2024-11-14T09:29:16,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741839_1015 (size=369) 2024-11-14T09:29:16,562 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T09:29:16,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741839_1015 (size=369) 2024-11-14T09:29:16,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741840_1016 (size=231) 2024-11-14T09:29:16,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741840_1016 (size=231) 2024-11-14T09:29:16,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741840_1016 (size=231) 2024-11-14T09:29:16,600 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-1, size=369 (369bytes) 2024-11-14T09:29:16,601 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-14T09:29:16,602 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-14T09:29:16,602 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-1 2024-11-14T09:29:16,608 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-1 after 4ms 2024-11-14T09:29:16,615 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-1 took 16ms 2024-11-14T09:29:16,620 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-1 so closing down 2024-11-14T09:29:16,627 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-14T09:29:16,627 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:16,629 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000001-wal-1.temp 2024-11-14T09:29:16,630 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:16,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741841_1017 (size=369) 2024-11-14T09:29:16,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741841_1017 (size=369) 2024-11-14T09:29:16,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741841_1017 (size=369) 2024-11-14T09:29:16,655 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:16,658 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000002 2024-11-14T09:29:16,663 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-14T09:29:16,664 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-1, journal: Splitting hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-1, size=369 (369bytes) at 1731576556601Finishing writing output for hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-1 so closing down at 1731576556620 (+19 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000001-wal-1.temp at 1731576556629 (+9 ms)3 split writer threads finished at 1731576556630 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731576556655 (+25 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000002 at 1731576556658 (+3 ms)Processed 2 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731576556664 (+6 ms) 2024-11-14T09:29:16,688 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-2, size=231 (231bytes) 2024-11-14T09:29:16,688 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-2 2024-11-14T09:29:16,689 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-2 after 1ms 2024-11-14T09:29:16,694 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-2 took 7ms 2024-11-14T09:29:16,699 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-2 so closing down 2024-11-14T09:29:16,699 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:16,704 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-14T09:29:16,706 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000002-wal-2.temp 2024-11-14T09:29:16,706 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:16,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741842_1018 (size=231) 2024-11-14T09:29:16,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741842_1018 (size=231) 2024-11-14T09:29:16,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741842_1018 (size=231) 2024-11-14T09:29:16,719 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:16,728 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-14T09:29:16,730 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-14T09:29:16,730 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-2, journal: Splitting hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-2, size=231 (231bytes) at 1731576556688Finishing writing output for hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-2 so closing down at 1731576556699 (+11 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000002-wal-2.temp at 1731576556706 (+7 ms)3 split writer threads finished at 1731576556706Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731576556719 (+13 ms)Processed 1 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731576556730 (+11 ms) 2024-11-14T09:29:16,730 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:16,733 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:16,752 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal.1731576556735, exclude list is [], retry=0 2024-11-14T09:29:16,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:16,758 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:16,758 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:16,761 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal.1731576556735 2024-11-14T09:29:16,761 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:16,761 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 63e0f39980e2ffcdf2d4594ca884d7df, NAME => 'testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:16,762 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:16,762 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,762 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,764 INFO [StoreOpener-63e0f39980e2ffcdf2d4594ca884d7df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,765 INFO [StoreOpener-63e0f39980e2ffcdf2d4594ca884d7df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63e0f39980e2ffcdf2d4594ca884d7df columnFamilyName a 2024-11-14T09:29:16,765 DEBUG [StoreOpener-63e0f39980e2ffcdf2d4594ca884d7df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:16,766 INFO [StoreOpener-63e0f39980e2ffcdf2d4594ca884d7df-1 {}] regionserver.HStore(327): Store=63e0f39980e2ffcdf2d4594ca884d7df/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:16,767 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,768 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,771 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:16,773 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000002 2024-11-14T09:29:16,784 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000002 2024-11-14T09:29:16,790 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 63e0f39980e2ffcdf2d4594ca884d7df 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-14T09:29:16,860 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/.tmp/a/844b5f0822944391b415687640128d89 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731576556519/Put/seqid=0 2024-11-14T09:29:16,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741844_1020 (size=5170) 2024-11-14T09:29:16,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741844_1020 (size=5170) 2024-11-14T09:29:16,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741844_1020 (size=5170) 2024-11-14T09:29:17,279 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/.tmp/a/844b5f0822944391b415687640128d89 2024-11-14T09:29:17,334 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/.tmp/a/844b5f0822944391b415687640128d89 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/a/844b5f0822944391b415687640128d89 2024-11-14T09:29:17,350 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/a/844b5f0822944391b415687640128d89, entries=2, sequenceid=2, filesize=5.0 K 2024-11-14T09:29:17,358 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 63e0f39980e2ffcdf2d4594ca884d7df in 567ms, sequenceid=2, compaction requested=false; wal=null 2024-11-14T09:29:17,360 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/0000000000000000002 2024-11-14T09:29:17,361 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:17,362 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:17,365 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 63e0f39980e2ffcdf2d4594ca884d7df 2024-11-14T09:29:17,368 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/63e0f39980e2ffcdf2d4594ca884d7df/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-14T09:29:17,370 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 63e0f39980e2ffcdf2d4594ca884d7df; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66087177, jitterRate=-0.01522432267665863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T09:29:17,370 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 63e0f39980e2ffcdf2d4594ca884d7df: Writing region info on filesystem at 1731576556762Initializing all the Stores at 1731576556763 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576556764 (+1 ms)Obtaining lock to block concurrent updates at 1731576556790 (+26 ms)Preparing flush snapshotting stores in 63e0f39980e2ffcdf2d4594ca884d7df at 1731576556790Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731576556794 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731576556372.63e0f39980e2ffcdf2d4594ca884d7df. at 1731576556794Flushing 63e0f39980e2ffcdf2d4594ca884d7df/a: creating writer at 1731576556796 (+2 ms)Flushing 63e0f39980e2ffcdf2d4594ca884d7df/a: appending metadata at 1731576556849 (+53 ms)Flushing 63e0f39980e2ffcdf2d4594ca884d7df/a: closing flushed file at 1731576556852 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d5e9e5f: reopening flushed file at 1731576557332 (+480 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 63e0f39980e2ffcdf2d4594ca884d7df in 567ms, sequenceid=2, compaction requested=false; wal=null at 1731576557358 (+26 ms)Cleaning up temporary data from old regions at 1731576557362 (+4 ms)Region opened successfully at 1731576557370 (+8 ms) 2024-11-14T09:29:17,398 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=373 (was 361) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:55736 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:55584 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:43606 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:43614 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:45840 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:45720 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:34615/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 278), ProcessCount=11 (was 11), AvailableMemoryMB=7001 (was 7082) 2024-11-14T09:29:17,411 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=373, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=7002 2024-11-14T09:29:17,428 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:17,430 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:17,431 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:17,436 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63513517, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/hregion-63513517, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:17,451 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63513517/hregion-63513517.1731576557437, exclude list is [], retry=0 2024-11-14T09:29:17,455 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:17,455 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:17,461 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:17,465 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63513517/hregion-63513517.1731576557437 2024-11-14T09:29:17,465 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:17,466 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e530f48cc8bdc6d71a318d82aa639860, NAME => 'testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/hbase 2024-11-14T09:29:17,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741846_1022 (size=64) 2024-11-14T09:29:17,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741846_1022 (size=64) 2024-11-14T09:29:17,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741846_1022 (size=64) 2024-11-14T09:29:17,479 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:17,481 INFO [StoreOpener-e530f48cc8bdc6d71a318d82aa639860-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,483 INFO [StoreOpener-e530f48cc8bdc6d71a318d82aa639860-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e530f48cc8bdc6d71a318d82aa639860 columnFamilyName a 2024-11-14T09:29:17,484 DEBUG [StoreOpener-e530f48cc8bdc6d71a318d82aa639860-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:17,484 INFO [StoreOpener-e530f48cc8bdc6d71a318d82aa639860-1 {}] regionserver.HStore(327): Store=e530f48cc8bdc6d71a318d82aa639860/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:17,485 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,486 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,486 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,487 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,487 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,490 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,494 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:17,494 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e530f48cc8bdc6d71a318d82aa639860; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73895557, jitterRate=0.10112960636615753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T09:29:17,495 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e530f48cc8bdc6d71a318d82aa639860: Writing region info on filesystem at 1731576557479Initializing all the Stores at 1731576557481 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576557481Cleaning up temporary data from old regions at 1731576557487 (+6 ms)Region opened successfully at 1731576557494 (+7 ms) 2024-11-14T09:29:17,495 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e530f48cc8bdc6d71a318d82aa639860, disabling compactions & flushes 2024-11-14T09:29:17,495 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860. 2024-11-14T09:29:17,495 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860. 2024-11-14T09:29:17,495 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860. after waiting 0 ms 2024-11-14T09:29:17,495 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860. 2024-11-14T09:29:17,496 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860. 2024-11-14T09:29:17,496 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e530f48cc8bdc6d71a318d82aa639860: Waiting for close lock at 1731576557495Disabling compacts and flushes for region at 1731576557495Disabling writes for close at 1731576557495Writing region close event to WAL at 1731576557496 (+1 ms)Closed at 1731576557496 2024-11-14T09:29:17,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741845_1021 (size=93) 2024-11-14T09:29:17,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741845_1021 (size=93) 2024-11-14T09:29:17,505 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-63513517/hregion-63513517.1731576557437 not finished, retry = 0 2024-11-14T09:29:17,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741845_1021 (size=93) 2024-11-14T09:29:17,610 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:17,610 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63513517:(num 1731576557437) 2024-11-14T09:29:17,611 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T09:29:17,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741847_1023 (size=369) 2024-11-14T09:29:17,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741847_1023 (size=369) 2024-11-14T09:29:17,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741847_1023 (size=369) 2024-11-14T09:29:17,627 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T09:29:17,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741848_1024 (size=231) 2024-11-14T09:29:17,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741848_1024 (size=231) 2024-11-14T09:29:17,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741848_1024 (size=231) 2024-11-14T09:29:17,658 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-2, size=231 (231bytes) 2024-11-14T09:29:17,658 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-2 2024-11-14T09:29:17,659 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-2 after 1ms 2024-11-14T09:29:17,663 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-2 took 6ms 2024-11-14T09:29:17,666 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-2 so closing down 2024-11-14T09:29:17,666 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:17,668 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-14T09:29:17,670 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002-wal-2.temp 2024-11-14T09:29:17,670 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:17,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741849_1025 (size=231) 2024-11-14T09:29:17,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741849_1025 (size=231) 2024-11-14T09:29:17,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741849_1025 (size=231) 2024-11-14T09:29:17,679 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:17,682 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002 2024-11-14T09:29:17,682 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-14T09:29:17,682 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-2, journal: Splitting hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-2, size=231 (231bytes) at 1731576557658Finishing writing output for hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-2 so closing down at 1731576557666 (+8 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002-wal-2.temp at 1731576557670 (+4 ms)3 split writer threads finished at 1731576557670Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731576557679 (+9 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002 at 1731576557682 (+3 ms)Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731576557682 2024-11-14T09:29:17,702 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-1, size=369 (369bytes) 2024-11-14T09:29:17,702 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-1 2024-11-14T09:29:17,703 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-1 after 1ms 2024-11-14T09:29:17,707 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-1 took 5ms 2024-11-14T09:29:17,709 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-1 so closing down 2024-11-14T09:29:17,710 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:17,712 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-14T09:29:17,714 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000001-wal-1.temp 2024-11-14T09:29:17,714 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:17,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741850_1026 (size=369) 2024-11-14T09:29:17,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741850_1026 (size=369) 2024-11-14T09:29:17,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741850_1026 (size=369) 2024-11-14T09:29:17,725 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:17,731 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002, length=231 2024-11-14T09:29:17,735 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002 2024-11-14T09:29:17,735 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-14T09:29:17,735 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-1, journal: Splitting hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-1, size=369 (369bytes) at 1731576557702Finishing writing output for hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-1 so closing down at 1731576557709 (+7 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000001-wal-1.temp at 1731576557714 (+5 ms)3 split writer threads finished at 1731576557714Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731576557725 (+11 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002 at 1731576557735 (+10 ms)Processed 2 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731576557735 2024-11-14T09:29:17,736 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:17,738 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:17,755 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal.1731576557739, exclude list is [], retry=0 2024-11-14T09:29:17,758 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:17,758 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:17,759 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:17,761 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal.1731576557739 2024-11-14T09:29:17,761 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:17,761 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e530f48cc8bdc6d71a318d82aa639860, NAME => 'testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:17,761 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:17,762 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,762 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,764 INFO [StoreOpener-e530f48cc8bdc6d71a318d82aa639860-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,766 INFO [StoreOpener-e530f48cc8bdc6d71a318d82aa639860-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e530f48cc8bdc6d71a318d82aa639860 columnFamilyName a 2024-11-14T09:29:17,766 DEBUG [StoreOpener-e530f48cc8bdc6d71a318d82aa639860-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:17,767 INFO [StoreOpener-e530f48cc8bdc6d71a318d82aa639860-1 {}] regionserver.HStore(327): Store=e530f48cc8bdc6d71a318d82aa639860/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:17,767 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,768 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,770 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,771 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002 2024-11-14T09:29:17,776 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002 2024-11-14T09:29:17,776 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e530f48cc8bdc6d71a318d82aa639860 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-14T09:29:17,800 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/.tmp/a/cc6fcf51979147e281c60343e327407e is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731576557610/Put/seqid=0 2024-11-14T09:29:17,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741852_1028 (size=5170) 2024-11-14T09:29:17,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741852_1028 (size=5170) 2024-11-14T09:29:17,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741852_1028 (size=5170) 2024-11-14T09:29:17,812 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/.tmp/a/cc6fcf51979147e281c60343e327407e 2024-11-14T09:29:17,824 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/.tmp/a/cc6fcf51979147e281c60343e327407e as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/a/cc6fcf51979147e281c60343e327407e 2024-11-14T09:29:17,835 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/a/cc6fcf51979147e281c60343e327407e, entries=2, sequenceid=2, filesize=5.0 K 2024-11-14T09:29:17,836 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e530f48cc8bdc6d71a318d82aa639860 in 60ms, sequenceid=2, compaction requested=false; wal=null 2024-11-14T09:29:17,837 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/0000000000000000002 2024-11-14T09:29:17,838 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,838 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,840 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e530f48cc8bdc6d71a318d82aa639860 2024-11-14T09:29:17,844 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/e530f48cc8bdc6d71a318d82aa639860/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-14T09:29:17,845 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e530f48cc8bdc6d71a318d82aa639860; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68589955, jitterRate=0.02206997573375702}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T09:29:17,846 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e530f48cc8bdc6d71a318d82aa639860: Writing region info on filesystem at 1731576557762Initializing all the Stores at 1731576557763 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576557763Obtaining lock to block concurrent updates at 1731576557776 (+13 ms)Preparing flush snapshotting stores in e530f48cc8bdc6d71a318d82aa639860 at 1731576557776Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731576557776Flushing stores of testReplayEditsWrittenIntoWAL,,1731576557429.e530f48cc8bdc6d71a318d82aa639860. at 1731576557776Flushing e530f48cc8bdc6d71a318d82aa639860/a: creating writer at 1731576557776Flushing e530f48cc8bdc6d71a318d82aa639860/a: appending metadata at 1731576557798 (+22 ms)Flushing e530f48cc8bdc6d71a318d82aa639860/a: closing flushed file at 1731576557798Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b81ee90: reopening flushed file at 1731576557823 (+25 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e530f48cc8bdc6d71a318d82aa639860 in 60ms, sequenceid=2, compaction requested=false; wal=null at 1731576557836 (+13 ms)Cleaning up temporary data from old regions at 1731576557838 (+2 ms)Region opened successfully at 1731576557845 (+7 ms) 2024-11-14T09:29:17,867 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=383 (was 373) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:43684 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:43606 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:45912 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:45720 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:55830 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 278), ProcessCount=11 (was 11), AvailableMemoryMB=6990 (was 7002) 2024-11-14T09:29:17,881 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=383, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=6990 2024-11-14T09:29:17,900 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:17,902 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:17,903 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:17,907 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-50908407, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/hregion-50908407, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:17,921 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-50908407/hregion-50908407.1731576557908, exclude list is [], retry=0 2024-11-14T09:29:17,925 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:17,925 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:17,929 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:17,934 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-50908407/hregion-50908407.1731576557908 2024-11-14T09:29:17,935 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:17,935 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 28ef2ebe9dae0a9769339033af15eccd, NAME => 'testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/hbase 2024-11-14T09:29:17,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741854_1030 (size=64) 2024-11-14T09:29:17,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741854_1030 (size=64) 2024-11-14T09:29:17,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741854_1030 (size=64) 2024-11-14T09:29:17,953 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:17,955 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:17,958 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ef2ebe9dae0a9769339033af15eccd columnFamilyName a 2024-11-14T09:29:17,958 DEBUG [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:17,958 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(327): Store=28ef2ebe9dae0a9769339033af15eccd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:17,958 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:17,960 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ef2ebe9dae0a9769339033af15eccd columnFamilyName b 2024-11-14T09:29:17,961 DEBUG [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:17,962 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(327): Store=28ef2ebe9dae0a9769339033af15eccd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:17,962 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:17,964 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ef2ebe9dae0a9769339033af15eccd columnFamilyName c 2024-11-14T09:29:17,964 DEBUG [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:17,965 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(327): Store=28ef2ebe9dae0a9769339033af15eccd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:17,965 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:17,966 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:17,966 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:17,969 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:17,969 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:17,970 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:17,972 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:17,976 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:17,976 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28ef2ebe9dae0a9769339033af15eccd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67127354, jitterRate=2.7552247047424316E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:17,977 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28ef2ebe9dae0a9769339033af15eccd: Writing region info on filesystem at 1731576557954Initializing all the Stores at 1731576557955 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576557955Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576557955Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576557955Cleaning up temporary data from old regions at 1731576557969 (+14 ms)Region opened successfully at 1731576557977 (+8 ms) 2024-11-14T09:29:17,977 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28ef2ebe9dae0a9769339033af15eccd, disabling compactions & flushes 2024-11-14T09:29:17,977 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. 2024-11-14T09:29:17,977 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. 2024-11-14T09:29:17,977 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. after waiting 0 ms 2024-11-14T09:29:17,977 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. 2024-11-14T09:29:17,978 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. 2024-11-14T09:29:17,978 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28ef2ebe9dae0a9769339033af15eccd: Waiting for close lock at 1731576557977Disabling compacts and flushes for region at 1731576557977Disabling writes for close at 1731576557977Writing region close event to WAL at 1731576557978 (+1 ms)Closed at 1731576557978 2024-11-14T09:29:17,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741853_1029 (size=93) 2024-11-14T09:29:17,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741853_1029 (size=93) 2024-11-14T09:29:17,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741853_1029 (size=93) 2024-11-14T09:29:17,985 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:17,985 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-50908407:(num 1731576557908) 2024-11-14T09:29:17,986 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:17,988 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:18,003 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989, exclude list is [], retry=0 2024-11-14T09:29:18,006 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:18,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:18,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:18,009 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 2024-11-14T09:29:18,010 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:18,252 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989, size=0 (0bytes) 2024-11-14T09:29:18,252 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 might be still open, length is 0 2024-11-14T09:29:18,252 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 2024-11-14T09:29:18,253 WARN [IPC Server handler 0 on default port 34615 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-14T09:29:18,254 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 after 2ms 2024-11-14T09:29:20,995 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:55864 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37113:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55864 dst: /127.0.0.1:37113 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37113 remote=/127.0.0.1:55864]. Total timeout mills is 60000, 57212 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:20,996 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:45946 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41175:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45946 dst: /127.0.0.1:41175 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:20,996 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:43716 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43439:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43716 dst: /127.0.0.1:43439 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:21,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741855_1032 (size=470675) 2024-11-14T09:29:21,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741855_1032 (size=470675) 2024-11-14T09:29:21,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741855_1032 (size=470675) 2024-11-14T09:29:21,424 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:29:21,471 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T09:29:22,255 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 after 4003ms 2024-11-14T09:29:22,260 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 took 4008ms 2024-11-14T09:29:22,265 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731576557989.temp 2024-11-14T09:29:22,268 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000000001-wal.1731576557989.temp 2024-11-14T09:29:22,343 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989; continuing. 2024-11-14T09:29:22,343 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 so closing down 2024-11-14T09:29:22,344 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:22,344 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:22,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741856_1033 (size=470683) 2024-11-14T09:29:22,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741856_1033 (size=470683) 2024-11-14T09:29:22,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741856_1033 (size=470683) 2024-11-14T09:29:22,349 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000000001-wal.1731576557989.temp (wrote 3002 edits, skipped 0 edits in 31 ms) 2024-11-14T09:29:22,351 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000000001-wal.1731576557989.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000003002 2024-11-14T09:29:22,351 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 90 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T09:29:22,351 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989, journal: Splitting hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989, size=0 (0bytes) at 1731576558252Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000000001-wal.1731576557989.temp at 1731576562268 (+4016 ms)Split 1024 edits, skipped 0 edits. at 1731576562309 (+41 ms)Split 2048 edits, skipped 0 edits. at 1731576562329 (+20 ms)Finishing writing output for hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 so closing down at 1731576562343 (+14 ms)3 split writer threads finished at 1731576562344 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000000001-wal.1731576557989.temp (wrote 3002 edits, skipped 0 edits in 31 ms) at 1731576562349 (+5 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000000001-wal.1731576557989.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000003002 at 1731576562351 (+2 ms)Processed 3002 edits across 1 Regions in 90 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989, size=0, length=0, corrupted=false, cancelled=false at 1731576562351 2024-11-14T09:29:22,355 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 to hdfs://localhost:34615/hbase/oldWALs/wal.1731576557989 2024-11-14T09:29:22,357 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000003002 2024-11-14T09:29:22,357 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:22,360 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:22,377 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576562360, exclude list is [], retry=0 2024-11-14T09:29:22,380 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:22,380 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:22,381 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:22,383 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576562360 2024-11-14T09:29:22,383 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:22,383 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:22,385 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:22,386 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ef2ebe9dae0a9769339033af15eccd columnFamilyName a 2024-11-14T09:29:22,386 DEBUG [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:22,387 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(327): Store=28ef2ebe9dae0a9769339033af15eccd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:22,387 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:22,388 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ef2ebe9dae0a9769339033af15eccd columnFamilyName b 2024-11-14T09:29:22,388 DEBUG [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:22,388 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(327): Store=28ef2ebe9dae0a9769339033af15eccd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:22,389 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:22,390 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ef2ebe9dae0a9769339033af15eccd columnFamilyName c 2024-11-14T09:29:22,390 DEBUG [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:22,390 INFO [StoreOpener-28ef2ebe9dae0a9769339033af15eccd-1 {}] regionserver.HStore(327): Store=28ef2ebe9dae0a9769339033af15eccd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:22,390 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:22,391 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:22,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:22,394 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000003002 2024-11-14T09:29:22,437 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T09:29:22,810 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28ef2ebe9dae0a9769339033af15eccd 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T09:29:22,847 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/a/6f664ee4a86641c2b2b1298a080c79dd is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731576558020/Put/seqid=0 2024-11-14T09:29:22,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741858_1035 (size=50463) 2024-11-14T09:29:22,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741858_1035 (size=50463) 2024-11-14T09:29:22,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741858_1035 (size=50463) 2024-11-14T09:29:22,859 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/a/6f664ee4a86641c2b2b1298a080c79dd 2024-11-14T09:29:22,867 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/a/6f664ee4a86641c2b2b1298a080c79dd as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/a/6f664ee4a86641c2b2b1298a080c79dd 2024-11-14T09:29:22,874 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/a/6f664ee4a86641c2b2b1298a080c79dd, entries=754, sequenceid=754, filesize=49.3 K 2024-11-14T09:29:22,875 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 28ef2ebe9dae0a9769339033af15eccd in 65ms, sequenceid=754, compaction requested=false; wal=null 2024-11-14T09:29:22,885 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T09:29:22,885 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28ef2ebe9dae0a9769339033af15eccd 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T09:29:22,895 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/a/89140232a46e482ead0aa8b26d9a2dea is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731576558070/Put/seqid=0 2024-11-14T09:29:22,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741859_1036 (size=20072) 2024-11-14T09:29:22,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741859_1036 (size=20072) 2024-11-14T09:29:22,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741859_1036 (size=20072) 2024-11-14T09:29:22,912 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/a/89140232a46e482ead0aa8b26d9a2dea 2024-11-14T09:29:22,942 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/b/b1297859f931474c9b920584431ca634 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731576558096/Put/seqid=0 2024-11-14T09:29:22,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741860_1037 (size=35835) 2024-11-14T09:29:22,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741860_1037 (size=35835) 2024-11-14T09:29:22,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741860_1037 (size=35835) 2024-11-14T09:29:22,952 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/b/b1297859f931474c9b920584431ca634 2024-11-14T09:29:22,960 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/a/89140232a46e482ead0aa8b26d9a2dea as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/a/89140232a46e482ead0aa8b26d9a2dea 2024-11-14T09:29:22,967 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/a/89140232a46e482ead0aa8b26d9a2dea, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-14T09:29:22,968 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/b/b1297859f931474c9b920584431ca634 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/b/b1297859f931474c9b920584431ca634 2024-11-14T09:29:22,976 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/b/b1297859f931474c9b920584431ca634, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-14T09:29:22,977 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 28ef2ebe9dae0a9769339033af15eccd in 92ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-14T09:29:22,989 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T09:29:22,989 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28ef2ebe9dae0a9769339033af15eccd 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T09:29:22,999 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/b/c87b57315123440a8198d4313a416e95 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731576558118/Put/seqid=0 2024-11-14T09:29:23,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741861_1038 (size=35082) 2024-11-14T09:29:23,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741861_1038 (size=35082) 2024-11-14T09:29:23,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741861_1038 (size=35082) 2024-11-14T09:29:23,022 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/b/c87b57315123440a8198d4313a416e95 2024-11-14T09:29:23,049 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/c/98e51c28f27542fe8cbea971f329c8a0 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731576558152/Put/seqid=0 2024-11-14T09:29:23,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741862_1039 (size=20825) 2024-11-14T09:29:23,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741862_1039 (size=20825) 2024-11-14T09:29:23,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741862_1039 (size=20825) 2024-11-14T09:29:23,060 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/c/98e51c28f27542fe8cbea971f329c8a0 2024-11-14T09:29:23,069 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/b/c87b57315123440a8198d4313a416e95 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/b/c87b57315123440a8198d4313a416e95 2024-11-14T09:29:23,078 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/b/c87b57315123440a8198d4313a416e95, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-14T09:29:23,080 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/c/98e51c28f27542fe8cbea971f329c8a0 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/c/98e51c28f27542fe8cbea971f329c8a0 2024-11-14T09:29:23,088 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/c/98e51c28f27542fe8cbea971f329c8a0, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-14T09:29:23,088 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 28ef2ebe9dae0a9769339033af15eccd in 99ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-14T09:29:23,096 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731576558205/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. 2024-11-14T09:29:23,100 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000003002 2024-11-14T09:29:23,101 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T09:29:23,101 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28ef2ebe9dae0a9769339033af15eccd 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-14T09:29:23,108 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/c/74b1e29c7c1d43ce9e62c7856a47fa06 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731576558161/Put/seqid=0 2024-11-14T09:29:23,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741863_1040 (size=50301) 2024-11-14T09:29:23,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741863_1040 (size=50301) 2024-11-14T09:29:23,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741863_1040 (size=50301) 2024-11-14T09:29:23,130 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/c/74b1e29c7c1d43ce9e62c7856a47fa06 2024-11-14T09:29:23,137 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 74b1e29c7c1d43ce9e62c7856a47fa06 2024-11-14T09:29:23,139 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/.tmp/c/74b1e29c7c1d43ce9e62c7856a47fa06 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/c/74b1e29c7c1d43ce9e62c7856a47fa06 2024-11-14T09:29:23,146 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 74b1e29c7c1d43ce9e62c7856a47fa06 2024-11-14T09:29:23,146 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/c/74b1e29c7c1d43ce9e62c7856a47fa06, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-14T09:29:23,146 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 28ef2ebe9dae0a9769339033af15eccd in 45ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-14T09:29:23,147 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/0000000000000003002 2024-11-14T09:29:23,148 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:23,148 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:23,149 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:29:23,151 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28ef2ebe9dae0a9769339033af15eccd 2024-11-14T09:29:23,154 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenIntoWAL/28ef2ebe9dae0a9769339033af15eccd/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-14T09:29:23,156 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28ef2ebe9dae0a9769339033af15eccd; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74834420, jitterRate=0.11511975526809692}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:29:23,156 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28ef2ebe9dae0a9769339033af15eccd: Writing region info on filesystem at 1731576562384Initializing all the Stores at 1731576562385 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576562385Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576562385Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576562385Cleaning up temporary data from old regions at 1731576563148 (+763 ms)Region opened successfully at 1731576563156 (+8 ms) 2024-11-14T09:29:23,227 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28ef2ebe9dae0a9769339033af15eccd, disabling compactions & flushes 2024-11-14T09:29:23,227 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. 2024-11-14T09:29:23,227 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. 2024-11-14T09:29:23,227 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. after waiting 0 ms 2024-11-14T09:29:23,227 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. 2024-11-14T09:29:23,231 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731576557900.28ef2ebe9dae0a9769339033af15eccd. 2024-11-14T09:29:23,231 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28ef2ebe9dae0a9769339033af15eccd: Waiting for close lock at 1731576563227Disabling compacts and flushes for region at 1731576563227Disabling writes for close at 1731576563227Writing region close event to WAL at 1731576563231 (+4 ms)Closed at 1731576563231 2024-11-14T09:29:23,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741857_1034 (size=93) 2024-11-14T09:29:23,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741857_1034 (size=93) 2024-11-14T09:29:23,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741857_1034 (size=93) 2024-11-14T09:29:23,240 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:23,240 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731576562360) 2024-11-14T09:29:23,254 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=395 (was 383) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1372905847_22 at /127.0.0.1:59698 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1372905847_22 at /127.0.0.1:56432 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1017487743) connection to localhost/127.0.0.1:45337 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:34615 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1372905847_22 at /127.0.0.1:59724 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1017487743) connection to localhost/127.0.0.1:44091 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1372905847_22 at /127.0.0.1:39754 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44091 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1017487743) connection to localhost/127.0.0.1:34615 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45337 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=264 (was 278), ProcessCount=11 (was 11), AvailableMemoryMB=7029 (was 6990) - AvailableMemoryMB LEAK? - 2024-11-14T09:29:23,266 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=395, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=264, ProcessCount=11, AvailableMemoryMB=7028 2024-11-14T09:29:23,283 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:23,286 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:23,287 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:23,289 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-93047873, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/hregion-93047873, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:23,303 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-93047873/hregion-93047873.1731576563290, exclude list is [], retry=0 2024-11-14T09:29:23,306 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:23,307 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:23,307 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:23,309 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-93047873/hregion-93047873.1731576563290 2024-11-14T09:29:23,310 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:23,310 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d67581a6246bf1ded199169c651a1bcd, NAME => 'test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/hbase 2024-11-14T09:29:23,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741865_1042 (size=43) 2024-11-14T09:29:23,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741865_1042 (size=43) 2024-11-14T09:29:23,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741865_1042 (size=43) 2024-11-14T09:29:23,321 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:23,323 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,325 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d67581a6246bf1ded199169c651a1bcd columnFamilyName a 2024-11-14T09:29:23,325 DEBUG [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:23,325 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(327): Store=d67581a6246bf1ded199169c651a1bcd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:23,326 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,327 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d67581a6246bf1ded199169c651a1bcd columnFamilyName b 2024-11-14T09:29:23,327 DEBUG [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:23,328 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(327): Store=d67581a6246bf1ded199169c651a1bcd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:23,328 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,330 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d67581a6246bf1ded199169c651a1bcd columnFamilyName c 2024-11-14T09:29:23,330 DEBUG [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:23,330 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(327): Store=d67581a6246bf1ded199169c651a1bcd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:23,330 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,331 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,332 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,333 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,333 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,334 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:23,335 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,338 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:23,338 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d67581a6246bf1ded199169c651a1bcd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69048349, jitterRate=0.02890057861804962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:23,340 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d67581a6246bf1ded199169c651a1bcd: Writing region info on filesystem at 1731576563321Initializing all the Stores at 1731576563322 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576563323 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576563323Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576563323Cleaning up temporary data from old regions at 1731576563333 (+10 ms)Region opened successfully at 1731576563340 (+7 ms) 2024-11-14T09:29:23,340 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d67581a6246bf1ded199169c651a1bcd, disabling compactions & flushes 2024-11-14T09:29:23,340 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. 2024-11-14T09:29:23,341 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. 2024-11-14T09:29:23,341 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. after waiting 0 ms 2024-11-14T09:29:23,341 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. 2024-11-14T09:29:23,341 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. 2024-11-14T09:29:23,341 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d67581a6246bf1ded199169c651a1bcd: Waiting for close lock at 1731576563340Disabling compacts and flushes for region at 1731576563340Disabling writes for close at 1731576563341 (+1 ms)Writing region close event to WAL at 1731576563341Closed at 1731576563341 2024-11-14T09:29:23,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741864_1041 (size=93) 2024-11-14T09:29:23,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741864_1041 (size=93) 2024-11-14T09:29:23,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741864_1041 (size=93) 2024-11-14T09:29:23,347 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:23,347 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-93047873:(num 1731576563290) 2024-11-14T09:29:23,347 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:23,350 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:23,364 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350, exclude list is [], retry=0 2024-11-14T09:29:23,367 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:23,368 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:23,368 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:23,370 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350 2024-11-14T09:29:23,370 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:23,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741866_1043 (size=344306) 2024-11-14T09:29:23,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741866_1043 (size=344306) 2024-11-14T09:29:23,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741866_1043 (size=344306) 2024-11-14T09:29:23,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T09:29:23,583 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T09:29:23,585 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T09:29:23,585 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T09:29:23,586 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T09:29:23,586 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T09:29:23,586 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350, size=336.2 K (344306bytes) 2024-11-14T09:29:23,586 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350 2024-11-14T09:29:23,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-14T09:29:23,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-14T09:29:23,588 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350 after 2ms 2024-11-14T09:29:23,593 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350 took 7ms 2024-11-14T09:29:23,597 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731576563350.temp 2024-11-14T09:29:23,599 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000000001-wal.1731576563350.temp 2024-11-14T09:29:23,631 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350 so closing down 2024-11-14T09:29:23,631 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:23,631 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:23,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741867_1044 (size=344306) 2024-11-14T09:29:23,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741867_1044 (size=344306) 2024-11-14T09:29:23,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741867_1044 (size=344306) 2024-11-14T09:29:23,638 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000000001-wal.1731576563350.temp (wrote 3000 edits, skipped 0 edits in 13 ms) 2024-11-14T09:29:23,640 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000000001-wal.1731576563350.temp to hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003000 2024-11-14T09:29:23,640 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-14T09:29:23,641 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350, journal: Splitting hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350, size=336.2 K (344306bytes) at 1731576563586Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000000001-wal.1731576563350.temp at 1731576563599 (+13 ms)Split 1024 edits, skipped 0 edits. at 1731576563606 (+7 ms)Split 2048 edits, skipped 0 edits. at 1731576563619 (+13 ms)Finishing writing output for hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350 so closing down at 1731576563631 (+12 ms)3 split writer threads finished at 1731576563631Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000000001-wal.1731576563350.temp (wrote 3000 edits, skipped 0 edits in 13 ms) at 1731576563638 (+7 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000000001-wal.1731576563350.temp to hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003000 at 1731576563640 (+2 ms)Processed 3000 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1731576563640 2024-11-14T09:29:23,643 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563350 to hdfs://localhost:34615/hbase/oldWALs/wal.1731576563350 2024-11-14T09:29:23,644 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003000 2024-11-14T09:29:23,644 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:23,646 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:23,661 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647, exclude list is [], retry=0 2024-11-14T09:29:23,665 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:23,665 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:23,666 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:23,673 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647 2024-11-14T09:29:23,673 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:23,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741868_1045 (size=344433) 2024-11-14T09:29:23,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741868_1045 (size=344433) 2024-11-14T09:29:23,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741868_1045 (size=344433) 2024-11-14T09:29:23,897 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647, size=336.4 K (344433bytes) 2024-11-14T09:29:23,898 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647 2024-11-14T09:29:23,898 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647 after 0ms 2024-11-14T09:29:23,902 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647 took 5ms 2024-11-14T09:29:23,906 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731576563647.temp 2024-11-14T09:29:23,907 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003001-wal.1731576563647.temp 2024-11-14T09:29:23,926 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647 so closing down 2024-11-14T09:29:23,926 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:23,927 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:23,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741869_1046 (size=344433) 2024-11-14T09:29:23,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741869_1046 (size=344433) 2024-11-14T09:29:23,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741869_1046 (size=344433) 2024-11-14T09:29:23,948 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003001-wal.1731576563647.temp (wrote 3000 edits, skipped 0 edits in 8 ms) 2024-11-14T09:29:23,950 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003001-wal.1731576563647.temp to hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000006000 2024-11-14T09:29:23,950 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-14T09:29:23,950 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647, journal: Splitting hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647, size=336.4 K (344433bytes) at 1731576563897Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003001-wal.1731576563647.temp at 1731576563907 (+10 ms)Split 1024 edits, skipped 0 edits. at 1731576563911 (+4 ms)Split 2048 edits, skipped 0 edits. at 1731576563919 (+8 ms)Finishing writing output for hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647 so closing down at 1731576563926 (+7 ms)3 split writer threads finished at 1731576563927 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003001-wal.1731576563647.temp (wrote 3000 edits, skipped 0 edits in 8 ms) at 1731576563948 (+21 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003001-wal.1731576563647.temp to hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000006000 at 1731576563950 (+2 ms)Processed 3000 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1731576563950 2024-11-14T09:29:23,952 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563647 to hdfs://localhost:34615/hbase/oldWALs/wal.1731576563647 2024-11-14T09:29:23,953 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000006000 2024-11-14T09:29:23,953 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:23,955 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/test2727-manual,16010,1731576563282, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:23,969 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563955, exclude list is [], retry=0 2024-11-14T09:29:23,973 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:23,973 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:23,974 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:23,982 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563955 2024-11-14T09:29:23,982 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:23,983 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d67581a6246bf1ded199169c651a1bcd, NAME => 'test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:23,983 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:23,983 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,983 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,985 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,987 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d67581a6246bf1ded199169c651a1bcd columnFamilyName a 2024-11-14T09:29:23,987 DEBUG [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:23,987 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(327): Store=d67581a6246bf1ded199169c651a1bcd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:23,988 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,989 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d67581a6246bf1ded199169c651a1bcd columnFamilyName b 2024-11-14T09:29:23,989 DEBUG [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:23,989 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(327): Store=d67581a6246bf1ded199169c651a1bcd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:23,989 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,990 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d67581a6246bf1ded199169c651a1bcd columnFamilyName c 2024-11-14T09:29:23,990 DEBUG [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:23,991 INFO [StoreOpener-d67581a6246bf1ded199169c651a1bcd-1 {}] regionserver.HStore(327): Store=d67581a6246bf1ded199169c651a1bcd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:23,991 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,992 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,994 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:23,995 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003000 2024-11-14T09:29:24,022 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003000 2024-11-14T09:29:24,024 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000006000 2024-11-14T09:29:24,047 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000006000 2024-11-14T09:29:24,048 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d67581a6246bf1ded199169c651a1bcd 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-14T09:29:24,068 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/.tmp/a/71363bc8ca2c415582c60b33bcf85c54 is 41, key is test2727/a:100/1731576563677/Put/seqid=0 2024-11-14T09:29:24,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741871_1048 (size=84227) 2024-11-14T09:29:24,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741871_1048 (size=84227) 2024-11-14T09:29:24,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741871_1048 (size=84227) 2024-11-14T09:29:24,080 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/.tmp/a/71363bc8ca2c415582c60b33bcf85c54 2024-11-14T09:29:24,109 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/.tmp/b/71fc5b0084814ae68a5fc66fad19f1cc is 41, key is test2727/b:100/1731576563729/Put/seqid=0 2024-11-14T09:29:24,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741872_1049 (size=84609) 2024-11-14T09:29:24,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741872_1049 (size=84609) 2024-11-14T09:29:24,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741872_1049 (size=84609) 2024-11-14T09:29:24,116 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/.tmp/b/71fc5b0084814ae68a5fc66fad19f1cc 2024-11-14T09:29:24,144 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/.tmp/c/fb00321fa9774a8cbf23385d19037594 is 41, key is test2727/c:100/1731576563776/Put/seqid=0 2024-11-14T09:29:24,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741873_1050 (size=84609) 2024-11-14T09:29:24,152 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/.tmp/c/fb00321fa9774a8cbf23385d19037594 2024-11-14T09:29:24,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741873_1050 (size=84609) 2024-11-14T09:29:24,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741873_1050 (size=84609) 2024-11-14T09:29:24,160 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/.tmp/a/71363bc8ca2c415582c60b33bcf85c54 as hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/a/71363bc8ca2c415582c60b33bcf85c54 2024-11-14T09:29:24,167 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/a/71363bc8ca2c415582c60b33bcf85c54, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-14T09:29:24,169 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/.tmp/b/71fc5b0084814ae68a5fc66fad19f1cc as hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/b/71fc5b0084814ae68a5fc66fad19f1cc 2024-11-14T09:29:24,175 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/b/71fc5b0084814ae68a5fc66fad19f1cc, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-14T09:29:24,176 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/.tmp/c/fb00321fa9774a8cbf23385d19037594 as hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/c/fb00321fa9774a8cbf23385d19037594 2024-11-14T09:29:24,186 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/c/fb00321fa9774a8cbf23385d19037594, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-14T09:29:24,186 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for d67581a6246bf1ded199169c651a1bcd in 138ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-14T09:29:24,187 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000003000 2024-11-14T09:29:24,188 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/0000000000000006000 2024-11-14T09:29:24,189 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:24,189 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:24,190 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:24,192 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d67581a6246bf1ded199169c651a1bcd 2024-11-14T09:29:24,195 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/test2727/d67581a6246bf1ded199169c651a1bcd/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-14T09:29:24,196 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d67581a6246bf1ded199169c651a1bcd; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60021946, jitterRate=-0.10560330748558044}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:24,196 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d67581a6246bf1ded199169c651a1bcd: Writing region info on filesystem at 1731576563984Initializing all the Stores at 1731576563985 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576563985Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576563985Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576563985Obtaining lock to block concurrent updates at 1731576564048 (+63 ms)Preparing flush snapshotting stores in d67581a6246bf1ded199169c651a1bcd at 1731576564048Finished memstore snapshotting test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731576564048Flushing stores of test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. at 1731576564048Flushing d67581a6246bf1ded199169c651a1bcd/a: creating writer at 1731576564048Flushing d67581a6246bf1ded199169c651a1bcd/a: appending metadata at 1731576564068 (+20 ms)Flushing d67581a6246bf1ded199169c651a1bcd/a: closing flushed file at 1731576564068Flushing d67581a6246bf1ded199169c651a1bcd/b: creating writer at 1731576564086 (+18 ms)Flushing d67581a6246bf1ded199169c651a1bcd/b: appending metadata at 1731576564107 (+21 ms)Flushing d67581a6246bf1ded199169c651a1bcd/b: closing flushed file at 1731576564107Flushing d67581a6246bf1ded199169c651a1bcd/c: creating writer at 1731576564124 (+17 ms)Flushing d67581a6246bf1ded199169c651a1bcd/c: appending metadata at 1731576564143 (+19 ms)Flushing d67581a6246bf1ded199169c651a1bcd/c: closing flushed file at 1731576564143Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50c3e285: reopening flushed file at 1731576564159 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@891a335: reopening flushed file at 1731576564168 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1333d0b4: reopening flushed file at 1731576564175 (+7 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for d67581a6246bf1ded199169c651a1bcd in 138ms, sequenceid=6000, compaction requested=false; wal=null at 1731576564186 (+11 ms)Cleaning up temporary data from old regions at 1731576564189 (+3 ms)Region opened successfully at 1731576564196 (+7 ms) 2024-11-14T09:29:24,198 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-14T09:29:24,198 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d67581a6246bf1ded199169c651a1bcd, disabling compactions & flushes 2024-11-14T09:29:24,198 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. 2024-11-14T09:29:24,198 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. 2024-11-14T09:29:24,198 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. after waiting 0 ms 2024-11-14T09:29:24,198 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. 2024-11-14T09:29:24,199 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731576563284.d67581a6246bf1ded199169c651a1bcd. 2024-11-14T09:29:24,199 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d67581a6246bf1ded199169c651a1bcd: Waiting for close lock at 1731576564198Disabling compacts and flushes for region at 1731576564198Disabling writes for close at 1731576564198Writing region close event to WAL at 1731576564199 (+1 ms)Closed at 1731576564199 2024-11-14T09:29:24,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741870_1047 (size=93) 2024-11-14T09:29:24,204 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1731576563282/wal.1731576563955 not finished, retry = 0 2024-11-14T09:29:24,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741870_1047 (size=93) 2024-11-14T09:29:24,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741870_1047 (size=93) 2024-11-14T09:29:24,308 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:24,308 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731576563955) 2024-11-14T09:29:24,325 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=401 (was 395) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:59698 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:39894 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:56510 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=264 (was 264), ProcessCount=11 (was 11), AvailableMemoryMB=6878 (was 7028) 2024-11-14T09:29:24,338 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=401, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=264, ProcessCount=11, AvailableMemoryMB=6877 2024-11-14T09:29:24,354 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:24,361 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:24,361 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731576564361 2024-11-14T09:29:24,369 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361 2024-11-14T09:29:24,371 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:24,372 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 42d044d489a608de5c1bffde53619948, NAME => 'testSequentialEditLogSeqNum,,1731576564355.42d044d489a608de5c1bffde53619948.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:24,372 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731576564355.42d044d489a608de5c1bffde53619948.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:24,372 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 42d044d489a608de5c1bffde53619948 2024-11-14T09:29:24,372 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 42d044d489a608de5c1bffde53619948 2024-11-14T09:29:24,373 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948 doesn't exist for region: 42d044d489a608de5c1bffde53619948 on table testSequentialEditLogSeqNum 2024-11-14T09:29:24,374 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 42d044d489a608de5c1bffde53619948 on table testSequentialEditLogSeqNum 2024-11-14T09:29:24,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741875_1052 (size=62) 2024-11-14T09:29:24,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741875_1052 (size=62) 2024-11-14T09:29:24,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741875_1052 (size=62) 2024-11-14T09:29:24,390 INFO [StoreOpener-42d044d489a608de5c1bffde53619948-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 42d044d489a608de5c1bffde53619948 2024-11-14T09:29:24,391 INFO [StoreOpener-42d044d489a608de5c1bffde53619948-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42d044d489a608de5c1bffde53619948 columnFamilyName a 2024-11-14T09:29:24,392 DEBUG [StoreOpener-42d044d489a608de5c1bffde53619948-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:24,392 INFO [StoreOpener-42d044d489a608de5c1bffde53619948-1 {}] regionserver.HStore(327): Store=42d044d489a608de5c1bffde53619948/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:24,392 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 42d044d489a608de5c1bffde53619948 2024-11-14T09:29:24,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948 2024-11-14T09:29:24,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948 2024-11-14T09:29:24,394 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 42d044d489a608de5c1bffde53619948 2024-11-14T09:29:24,394 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 42d044d489a608de5c1bffde53619948 2024-11-14T09:29:24,396 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 42d044d489a608de5c1bffde53619948 2024-11-14T09:29:24,398 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:24,399 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 42d044d489a608de5c1bffde53619948; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59510924, jitterRate=-0.11321812868118286}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T09:29:24,399 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 42d044d489a608de5c1bffde53619948: Writing region info on filesystem at 1731576564372Initializing all the Stores at 1731576564390 (+18 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576564390Cleaning up temporary data from old regions at 1731576564394 (+4 ms)Region opened successfully at 1731576564399 (+5 ms) 2024-11-14T09:29:24,419 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 42d044d489a608de5c1bffde53619948 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-14T09:29:24,449 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/.tmp/a/7a7cd78d6e9e4b34942a9d6fe456b60e is 81, key is testSequentialEditLogSeqNum/a:x0/1731576564400/Put/seqid=0 2024-11-14T09:29:24,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741876_1053 (size=5833) 2024-11-14T09:29:24,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741876_1053 (size=5833) 2024-11-14T09:29:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741876_1053 (size=5833) 2024-11-14T09:29:24,458 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/.tmp/a/7a7cd78d6e9e4b34942a9d6fe456b60e 2024-11-14T09:29:24,466 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/.tmp/a/7a7cd78d6e9e4b34942a9d6fe456b60e as hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/a/7a7cd78d6e9e4b34942a9d6fe456b60e 2024-11-14T09:29:24,472 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/a/7a7cd78d6e9e4b34942a9d6fe456b60e, entries=10, sequenceid=13, filesize=5.7 K 2024-11-14T09:29:24,474 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 42d044d489a608de5c1bffde53619948 in 55ms, sequenceid=13, compaction requested=false 2024-11-14T09:29:24,474 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 42d044d489a608de5c1bffde53619948: 2024-11-14T09:29:24,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T09:29:24,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T09:29:24,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T09:29:24,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T09:29:24,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T09:29:24,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741874_1051 (size=3017) 2024-11-14T09:29:24,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741874_1051 (size=3017) 2024-11-14T09:29:24,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741874_1051 (size=3017) 2024-11-14T09:29:24,506 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361, size=2.9 K (3017bytes) 2024-11-14T09:29:24,506 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361 2024-11-14T09:29:24,506 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361 after 0ms 2024-11-14T09:29:24,510 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361 took 4ms 2024-11-14T09:29:24,513 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361 so closing down 2024-11-14T09:29:24,513 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:24,515 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731576564361.temp 2024-11-14T09:29:24,517 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/recovered.edits/0000000000000000003-wal.1731576564361.temp 2024-11-14T09:29:24,517 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:24,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741877_1054 (size=2433) 2024-11-14T09:29:24,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741877_1054 (size=2433) 2024-11-14T09:29:24,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741877_1054 (size=2433) 2024-11-14T09:29:24,533 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/recovered.edits/0000000000000000003-wal.1731576564361.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:24,535 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/recovered.edits/0000000000000000003-wal.1731576564361.temp to hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/recovered.edits/0000000000000000020 2024-11-14T09:29:24,535 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 24 ms; skipped=2; WAL=hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-14T09:29:24,535 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361, journal: Splitting hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361, size=2.9 K (3017bytes) at 1731576564506Finishing writing output for hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361 so closing down at 1731576564513 (+7 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/recovered.edits/0000000000000000003-wal.1731576564361.temp at 1731576564517 (+4 ms)3 split writer threads finished at 1731576564517Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/recovered.edits/0000000000000000003-wal.1731576564361.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731576564533 (+16 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/recovered.edits/0000000000000000003-wal.1731576564361.temp to hdfs://localhost:34615/hbase/data/default/testSequentialEditLogSeqNum/42d044d489a608de5c1bffde53619948/recovered.edits/0000000000000000020 at 1731576564535 (+2 ms)Processed 17 edits across 1 Regions in 24 ms; skipped=2; WAL=hdfs://localhost:34615/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731576564353/wal.1731576564361, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1731576564535 2024-11-14T09:29:24,563 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=406 (was 401) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:39894 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:56510 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=264 (was 264), ProcessCount=11 (was 11), AvailableMemoryMB=6877 (was 6877) 2024-11-14T09:29:24,579 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=406, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=264, ProcessCount=11, AvailableMemoryMB=6876 2024-11-14T09:29:24,602 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:24,605 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:24,633 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:24,636 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-28998638, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/hregion-28998638, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:24,650 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-28998638/hregion-28998638.1731576564636, exclude list is [], retry=0 2024-11-14T09:29:24,653 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:24,653 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:24,654 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:24,656 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-28998638/hregion-28998638.1731576564636 2024-11-14T09:29:24,656 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:24,657 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0c4924752b0e00ec2df2a7d5e98a1ffd, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/hbase 2024-11-14T09:29:24,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741879_1056 (size=70) 2024-11-14T09:29:24,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741879_1056 (size=70) 2024-11-14T09:29:24,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741879_1056 (size=70) 2024-11-14T09:29:24,669 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:24,671 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,674 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c4924752b0e00ec2df2a7d5e98a1ffd columnFamilyName a 2024-11-14T09:29:24,674 DEBUG [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:24,675 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(327): Store=0c4924752b0e00ec2df2a7d5e98a1ffd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:24,675 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,677 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c4924752b0e00ec2df2a7d5e98a1ffd columnFamilyName b 2024-11-14T09:29:24,677 DEBUG [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:24,677 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(327): Store=0c4924752b0e00ec2df2a7d5e98a1ffd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:24,677 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,679 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c4924752b0e00ec2df2a7d5e98a1ffd columnFamilyName c 2024-11-14T09:29:24,679 DEBUG [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:24,679 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(327): Store=0c4924752b0e00ec2df2a7d5e98a1ffd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:24,679 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,680 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,681 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,682 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,682 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,682 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:24,683 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,686 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:24,687 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0c4924752b0e00ec2df2a7d5e98a1ffd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60142471, jitterRate=-0.10380734503269196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:24,688 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0c4924752b0e00ec2df2a7d5e98a1ffd: Writing region info on filesystem at 1731576564669Initializing all the Stores at 1731576564671 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576564671Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576564671Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576564671Cleaning up temporary data from old regions at 1731576564682 (+11 ms)Region opened successfully at 1731576564688 (+6 ms) 2024-11-14T09:29:24,688 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0c4924752b0e00ec2df2a7d5e98a1ffd, disabling compactions & flushes 2024-11-14T09:29:24,689 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. 2024-11-14T09:29:24,689 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. 2024-11-14T09:29:24,689 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. after waiting 0 ms 2024-11-14T09:29:24,689 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. 2024-11-14T09:29:24,689 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. 2024-11-14T09:29:24,689 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0c4924752b0e00ec2df2a7d5e98a1ffd: Waiting for close lock at 1731576564688Disabling compacts and flushes for region at 1731576564688Disabling writes for close at 1731576564689 (+1 ms)Writing region close event to WAL at 1731576564689Closed at 1731576564689 2024-11-14T09:29:24,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741878_1055 (size=93) 2024-11-14T09:29:24,697 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-28998638/hregion-28998638.1731576564636 not finished, retry = 0 2024-11-14T09:29:24,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741878_1055 (size=93) 2024-11-14T09:29:24,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741878_1055 (size=93) 2024-11-14T09:29:24,801 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:24,801 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-28998638:(num 1731576564636) 2024-11-14T09:29:24,801 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:24,803 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:24,825 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804, exclude list is [], retry=0 2024-11-14T09:29:24,828 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:24,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:24,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:24,832 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 2024-11-14T09:29:24,832 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:24,832 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0c4924752b0e00ec2df2a7d5e98a1ffd, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:24,833 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:24,833 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,833 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,839 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,840 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c4924752b0e00ec2df2a7d5e98a1ffd columnFamilyName a 2024-11-14T09:29:24,840 DEBUG [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:24,841 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(327): Store=0c4924752b0e00ec2df2a7d5e98a1ffd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:24,841 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,842 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c4924752b0e00ec2df2a7d5e98a1ffd columnFamilyName b 2024-11-14T09:29:24,842 DEBUG [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:24,843 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(327): Store=0c4924752b0e00ec2df2a7d5e98a1ffd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:24,843 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,844 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c4924752b0e00ec2df2a7d5e98a1ffd columnFamilyName c 2024-11-14T09:29:24,844 DEBUG [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:24,844 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(327): Store=0c4924752b0e00ec2df2a7d5e98a1ffd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:24,845 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,845 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,846 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,848 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,848 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,848 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:24,849 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:24,850 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0c4924752b0e00ec2df2a7d5e98a1ffd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72954380, jitterRate=0.08710497617721558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:24,851 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0c4924752b0e00ec2df2a7d5e98a1ffd: Writing region info on filesystem at 1731576564837Initializing all the Stores at 1731576564838 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576564838Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576564839 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576564839Cleaning up temporary data from old regions at 1731576564848 (+9 ms)Region opened successfully at 1731576564851 (+3 ms) 2024-11-14T09:29:24,856 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731576564855/Put/seqid=0 2024-11-14T09:29:24,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741881_1058 (size=4826) 2024-11-14T09:29:24,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741881_1058 (size=4826) 2024-11-14T09:29:24,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741881_1058 (size=4826) 2024-11-14T09:29:24,865 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34615/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 0c4924752b0e00ec2df2a7d5e98a1ffd/a 2024-11-14T09:29:24,874 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-14T09:29:24,874 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T09:29:24,874 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0c4924752b0e00ec2df2a7d5e98a1ffd: 2024-11-14T09:29:24,876 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/a/6ab841621324450ab66ae9963da5e324_SeqId_3_ 2024-11-14T09:29:24,877 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34615/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 0c4924752b0e00ec2df2a7d5e98a1ffd/a as hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/a/6ab841621324450ab66ae9963da5e324_SeqId_3_ - updating store file list. 2024-11-14T09:29:24,883 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6ab841621324450ab66ae9963da5e324_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-14T09:29:24,884 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/a/6ab841621324450ab66ae9963da5e324_SeqId_3_ into 0c4924752b0e00ec2df2a7d5e98a1ffd/a 2024-11-14T09:29:24,884 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34615/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 0c4924752b0e00ec2df2a7d5e98a1ffd/a (new location: hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/a/6ab841621324450ab66ae9963da5e324_SeqId_3_) 2024-11-14T09:29:24,934 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804, size=0 (0bytes) 2024-11-14T09:29:24,934 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 might be still open, length is 0 2024-11-14T09:29:24,934 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 2024-11-14T09:29:24,935 WARN [IPC Server handler 1 on default port 34615 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-14T09:29:24,935 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 after 1ms 2024-11-14T09:29:26,987 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:56584 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37113:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56584 dst: /127.0.0.1:37113 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37113 remote=/127.0.0.1:56584]. Total timeout mills is 60000, 57905 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:26,987 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:39982 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41175:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39982 dst: /127.0.0.1:41175 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:26,987 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:59916 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43439:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59916 dst: /127.0.0.1:43439 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:26,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741880_1059 (size=504) 2024-11-14T09:29:26,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741880_1059 (size=504) 2024-11-14T09:29:28,936 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 after 4002ms 2024-11-14T09:29:28,939 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 took 4005ms 2024-11-14T09:29:28,941 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804; continuing. 2024-11-14T09:29:28,941 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 so closing down 2024-11-14T09:29:28,941 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:28,943 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731576564804.temp 2024-11-14T09:29:28,944 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005-wal.1731576564804.temp 2024-11-14T09:29:28,944 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:28,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741882_1060 (size=238) 2024-11-14T09:29:28,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741882_1060 (size=238) 2024-11-14T09:29:28,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741882_1060 (size=238) 2024-11-14T09:29:29,352 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005-wal.1731576564804.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:29,353 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005-wal.1731576564804.temp to hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005 2024-11-14T09:29:29,354 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 414 ms; skipped=1; WAL=hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T09:29:29,354 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804, journal: Splitting hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804, size=0 (0bytes) at 1731576564934Finishing writing output for hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 so closing down at 1731576568941 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005-wal.1731576564804.temp at 1731576568944 (+3 ms)3 split writer threads finished at 1731576568944Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005-wal.1731576564804.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731576569352 (+408 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005-wal.1731576564804.temp to hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005 at 1731576569353 (+1 ms)Processed 2 edits across 1 Regions in 414 ms; skipped=1; WAL=hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804, size=0, length=0, corrupted=false, cancelled=false at 1731576569354 (+1 ms) 2024-11-14T09:29:29,356 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 to hdfs://localhost:34615/hbase/oldWALs/wal.1731576564804 2024-11-14T09:29:29,357 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005 2024-11-14T09:29:29,357 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:29,358 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:29,372 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576569359, exclude list is [], retry=0 2024-11-14T09:29:29,375 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:29,376 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:29,376 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:29,378 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576569359 2024-11-14T09:29:29,378 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:29,378 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0c4924752b0e00ec2df2a7d5e98a1ffd, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:29,379 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:29,379 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,379 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,380 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,381 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c4924752b0e00ec2df2a7d5e98a1ffd columnFamilyName a 2024-11-14T09:29:29,382 DEBUG [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:29,394 DEBUG [StoreFileOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6ab841621324450ab66ae9963da5e324_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-14T09:29:29,394 DEBUG [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/a/6ab841621324450ab66ae9963da5e324_SeqId_3_ 2024-11-14T09:29:29,394 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(327): Store=0c4924752b0e00ec2df2a7d5e98a1ffd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:29,394 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,396 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c4924752b0e00ec2df2a7d5e98a1ffd columnFamilyName b 2024-11-14T09:29:29,396 DEBUG [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:29,396 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(327): Store=0c4924752b0e00ec2df2a7d5e98a1ffd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:29,397 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,398 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c4924752b0e00ec2df2a7d5e98a1ffd columnFamilyName c 2024-11-14T09:29:29,398 DEBUG [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:29,398 INFO [StoreOpener-0c4924752b0e00ec2df2a7d5e98a1ffd-1 {}] regionserver.HStore(327): Store=0c4924752b0e00ec2df2a7d5e98a1ffd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:29,398 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,399 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,401 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,402 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005 2024-11-14T09:29:29,405 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005 2024-11-14T09:29:29,405 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0c4924752b0e00ec2df2a7d5e98a1ffd 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-14T09:29:29,422 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/.tmp/a/316dfb828e534323953f627f411812be is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731576564890/Put/seqid=0 2024-11-14T09:29:29,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741884_1062 (size=5149) 2024-11-14T09:29:29,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741884_1062 (size=5149) 2024-11-14T09:29:29,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741884_1062 (size=5149) 2024-11-14T09:29:29,429 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/.tmp/a/316dfb828e534323953f627f411812be 2024-11-14T09:29:29,436 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/.tmp/a/316dfb828e534323953f627f411812be as hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/a/316dfb828e534323953f627f411812be 2024-11-14T09:29:29,441 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/a/316dfb828e534323953f627f411812be, entries=1, sequenceid=5, filesize=5.0 K 2024-11-14T09:29:29,442 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 0c4924752b0e00ec2df2a7d5e98a1ffd in 37ms, sequenceid=5, compaction requested=false; wal=null 2024-11-14T09:29:29,442 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/0000000000000000005 2024-11-14T09:29:29,444 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,444 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,445 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:29,446 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0c4924752b0e00ec2df2a7d5e98a1ffd 2024-11-14T09:29:29,448 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/0c4924752b0e00ec2df2a7d5e98a1ffd/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-14T09:29:29,449 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0c4924752b0e00ec2df2a7d5e98a1ffd; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60879462, jitterRate=-0.09282532334327698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:29,450 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0c4924752b0e00ec2df2a7d5e98a1ffd: Writing region info on filesystem at 1731576569379Initializing all the Stores at 1731576569380 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576569380Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576569380Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576569380Obtaining lock to block concurrent updates at 1731576569405 (+25 ms)Preparing flush snapshotting stores in 0c4924752b0e00ec2df2a7d5e98a1ffd at 1731576569405Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731576569405Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. at 1731576569405Flushing 0c4924752b0e00ec2df2a7d5e98a1ffd/a: creating writer at 1731576569405Flushing 0c4924752b0e00ec2df2a7d5e98a1ffd/a: appending metadata at 1731576569421 (+16 ms)Flushing 0c4924752b0e00ec2df2a7d5e98a1ffd/a: closing flushed file at 1731576569421Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c0cb120: reopening flushed file at 1731576569435 (+14 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 0c4924752b0e00ec2df2a7d5e98a1ffd in 37ms, sequenceid=5, compaction requested=false; wal=null at 1731576569442 (+7 ms)Cleaning up temporary data from old regions at 1731576569444 (+2 ms)Region opened successfully at 1731576569449 (+5 ms) 2024-11-14T09:29:29,454 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0c4924752b0e00ec2df2a7d5e98a1ffd, disabling compactions & flushes 2024-11-14T09:29:29,454 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. 2024-11-14T09:29:29,454 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. 2024-11-14T09:29:29,454 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. after waiting 0 ms 2024-11-14T09:29:29,454 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. 2024-11-14T09:29:29,455 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731576564603.0c4924752b0e00ec2df2a7d5e98a1ffd. 2024-11-14T09:29:29,455 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0c4924752b0e00ec2df2a7d5e98a1ffd: Waiting for close lock at 1731576569454Disabling compacts and flushes for region at 1731576569454Disabling writes for close at 1731576569454Writing region close event to WAL at 1731576569455 (+1 ms)Closed at 1731576569455 2024-11-14T09:29:29,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741883_1061 (size=93) 2024-11-14T09:29:29,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741883_1061 (size=93) 2024-11-14T09:29:29,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741883_1061 (size=93) 2024-11-14T09:29:29,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:29,461 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731576569359) 2024-11-14T09:29:29,478 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409 (was 406) Potentially hanging thread: IPC Client (1017487743) connection to localhost/127.0.0.1:34615 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-191416337_22 at /127.0.0.1:59948 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-191416337_22 at /127.0.0.1:40004 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:34615 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1021 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=258 (was 264), ProcessCount=11 (was 11), AvailableMemoryMB=6829 (was 6876) 2024-11-14T09:29:29,490 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409, OpenFileDescriptor=1021, MaxFileDescriptor=1048576, SystemLoadAverage=258, ProcessCount=11, AvailableMemoryMB=6827 2024-11-14T09:29:29,506 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:29,510 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:29:29,515 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 83f56b55f2af,43303,1731576553096 2024-11-14T09:29:29,517 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13f72017 2024-11-14T09:29:29,518 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:29:29,520 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:29:29,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:29:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-14T09:29:29,532 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:29:29,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-14T09:29:29,535 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:29,536 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:29:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:29:29,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741885_1063 (size=694) 2024-11-14T09:29:29,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741885_1063 (size=694) 2024-11-14T09:29:29,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741885_1063 (size=694) 2024-11-14T09:29:29,550 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e5dc7f00d707752df5f2aa388572824b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963 2024-11-14T09:29:29,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741886_1064 (size=77) 2024-11-14T09:29:29,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741886_1064 (size=77) 2024-11-14T09:29:29,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741886_1064 (size=77) 2024-11-14T09:29:29,559 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:29,559 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing e5dc7f00d707752df5f2aa388572824b, disabling compactions & flushes 2024-11-14T09:29:29,559 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:29,559 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:29,559 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. after waiting 0 ms 2024-11-14T09:29:29,559 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:29,559 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:29,559 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for e5dc7f00d707752df5f2aa388572824b: Waiting for close lock at 1731576569559Disabling compacts and flushes for region at 1731576569559Disabling writes for close at 1731576569559Writing region close event to WAL at 1731576569559Closed at 1731576569559 2024-11-14T09:29:29,561 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:29:29,568 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731576569561"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731576569561"}]},"ts":"1731576569561"} 2024-11-14T09:29:29,572 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:29:29,573 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:29:29,577 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731576569574"}]},"ts":"1731576569574"} 2024-11-14T09:29:29,581 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-14T09:29:29,582 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {83f56b55f2af=0} racks are {/default-rack=0} 2024-11-14T09:29:29,583 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T09:29:29,583 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T09:29:29,583 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T09:29:29,583 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T09:29:29,583 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T09:29:29,583 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T09:29:29,583 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T09:29:29,583 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T09:29:29,583 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T09:29:29,583 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T09:29:29,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, ASSIGN}] 2024-11-14T09:29:29,587 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, ASSIGN 2024-11-14T09:29:29,588 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, ASSIGN; state=OFFLINE, location=83f56b55f2af,35921,1731576554024; forceNewPlan=false, retain=false 2024-11-14T09:29:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:29:29,741 INFO [83f56b55f2af:43303 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T09:29:29,742 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=OPENING, regionLocation=83f56b55f2af,35921,1731576554024 2024-11-14T09:29:29,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, ASSIGN because future has completed 2024-11-14T09:29:29,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,35921,1731576554024}] 2024-11-14T09:29:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:29:29,906 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:29,906 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e5dc7f00d707752df5f2aa388572824b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:29,907 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,907 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:29,907 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,907 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,909 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,911 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf1 2024-11-14T09:29:29,911 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:29,911 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:29,912 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,913 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf2 2024-11-14T09:29:29,913 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:29,913 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:29,913 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,914 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,915 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,916 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,916 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,917 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T09:29:29,919 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,921 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:29,922 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e5dc7f00d707752df5f2aa388572824b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59099670, jitterRate=-0.11934629082679749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T09:29:29,922 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:29,922 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e5dc7f00d707752df5f2aa388572824b: Running coprocessor pre-open hook at 1731576569908Writing region info on filesystem at 1731576569908Initializing all the Stores at 1731576569909 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576569909Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576569909Cleaning up temporary data from old regions at 1731576569916 (+7 ms)Running coprocessor post-open hooks at 1731576569922 (+6 ms)Region opened successfully at 1731576569922 2024-11-14T09:29:29,924 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., pid=6, masterSystemTime=1731576569899 2024-11-14T09:29:29,928 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:29,928 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:29,929 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=OPEN, openSeqNum=2, regionLocation=83f56b55f2af,35921,1731576554024 2024-11-14T09:29:29,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,35921,1731576554024 because future has completed 2024-11-14T09:29:29,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T09:29:29,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,35921,1731576554024 in 187 msec 2024-11-14T09:29:29,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T09:29:29,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, ASSIGN in 351 msec 2024-11-14T09:29:29,940 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:29:29,941 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731576569941"}]},"ts":"1731576569941"} 2024-11-14T09:29:29,944 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-14T09:29:29,945 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:29:29,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 419 msec 2024-11-14T09:29:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:29:30,167 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-14T09:29:30,167 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-14T09:29:30,168 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T09:29:30,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-14T09:29:30,173 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T09:29:30,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-14T09:29:30,185 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=2] 2024-11-14T09:29:30,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e5dc7f00d707752df5f2aa388572824b, source=83f56b55f2af,35921,1731576554024, destination=83f56b55f2af,32943,1731576553877, warming up region on 83f56b55f2af,32943,1731576553877 2024-11-14T09:29:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T09:29:30,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e5dc7f00d707752df5f2aa388572824b, source=83f56b55f2af,35921,1731576554024, destination=83f56b55f2af,32943,1731576553877, running balancer 2024-11-14T09:29:30,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58915, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T09:29:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE 2024-11-14T09:29:30,206 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE 2024-11-14T09:29:30,208 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=CLOSING, regionLocation=83f56b55f2af,35921,1731576554024 2024-11-14T09:29:30,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] regionserver.HRegion(7855): Warmup {ENCODED => e5dc7f00d707752df5f2aa388572824b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:30,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE because future has completed 2024-11-14T09:29:30,212 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,213 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:29:30,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,35921,1731576554024}] 2024-11-14T09:29:30,214 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf1 2024-11-14T09:29:30,214 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:30,215 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:30,215 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,216 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf2 2024-11-14T09:29:30,216 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:30,217 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] regionserver.HRegion(1722): Closing e5dc7f00d707752df5f2aa388572824b, disabling compactions & flushes 2024-11-14T09:29:30,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. after waiting 0 ms 2024-11-14T09:29:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] regionserver.HRegion(1676): Region close journal for e5dc7f00d707752df5f2aa388572824b: Waiting for close lock at 1731576570217Disabling compacts and flushes for region at 1731576570217Disabling writes for close at 1731576570217Writing region close event to WAL at 1731576570217Closed at 1731576570217 2024-11-14T09:29:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-14T09:29:30,374 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,374 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:29:30,375 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing e5dc7f00d707752df5f2aa388572824b, disabling compactions & flushes 2024-11-14T09:29:30,375 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,375 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,375 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. after waiting 0 ms 2024-11-14T09:29:30,375 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,375 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing e5dc7f00d707752df5f2aa388572824b 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-14T09:29:30,393 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf1/a7dbe2683d5645cea14edbc20111c949 is 35, key is r1/cf1:q/1731576570187/Put/seqid=0 2024-11-14T09:29:30,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741887_1065 (size=4783) 2024-11-14T09:29:30,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741887_1065 (size=4783) 2024-11-14T09:29:30,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741887_1065 (size=4783) 2024-11-14T09:29:30,404 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf1/a7dbe2683d5645cea14edbc20111c949 2024-11-14T09:29:30,413 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf1/a7dbe2683d5645cea14edbc20111c949 as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/a7dbe2683d5645cea14edbc20111c949 2024-11-14T09:29:30,421 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/a7dbe2683d5645cea14edbc20111c949, entries=1, sequenceid=5, filesize=4.7 K 2024-11-14T09:29:30,423 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for e5dc7f00d707752df5f2aa388572824b in 48ms, sequenceid=5, compaction requested=false 2024-11-14T09:29:30,423 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-14T09:29:30,429 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-14T09:29:30,433 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,433 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for e5dc7f00d707752df5f2aa388572824b: Waiting for close lock at 1731576570375Running coprocessor pre-close hooks at 1731576570375Disabling compacts and flushes for region at 1731576570375Disabling writes for close at 1731576570375Obtaining lock to block concurrent updates at 1731576570375Preparing flush snapshotting stores in e5dc7f00d707752df5f2aa388572824b at 1731576570375Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731576570376 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. at 1731576570376Flushing e5dc7f00d707752df5f2aa388572824b/cf1: creating writer at 1731576570376Flushing e5dc7f00d707752df5f2aa388572824b/cf1: appending metadata at 1731576570393 (+17 ms)Flushing e5dc7f00d707752df5f2aa388572824b/cf1: closing flushed file at 1731576570393Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67dac3af: reopening flushed file at 1731576570412 (+19 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for e5dc7f00d707752df5f2aa388572824b in 48ms, sequenceid=5, compaction requested=false at 1731576570423 (+11 ms)Writing region close event to WAL at 1731576570425 (+2 ms)Running coprocessor post-close hooks at 1731576570430 (+5 ms)Closed at 1731576570433 (+3 ms) 2024-11-14T09:29:30,433 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding e5dc7f00d707752df5f2aa388572824b move to 83f56b55f2af,32943,1731576553877 record at close sequenceid=5 2024-11-14T09:29:30,437 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,439 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=CLOSED 2024-11-14T09:29:30,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,35921,1731576554024 because future has completed 2024-11-14T09:29:30,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T09:29:30,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,35921,1731576554024 in 229 msec 2024-11-14T09:29:30,451 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE; state=CLOSED, location=83f56b55f2af,32943,1731576553877; forceNewPlan=false, retain=false 2024-11-14T09:29:30,602 INFO [83f56b55f2af:43303 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T09:29:30,602 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=OPENING, regionLocation=83f56b55f2af,32943,1731576553877 2024-11-14T09:29:30,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE because future has completed 2024-11-14T09:29:30,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,32943,1731576553877}] 2024-11-14T09:29:30,763 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,763 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e5dc7f00d707752df5f2aa388572824b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:30,764 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,764 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:30,764 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,764 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,765 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,766 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf1 2024-11-14T09:29:30,766 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:30,773 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/a7dbe2683d5645cea14edbc20111c949 2024-11-14T09:29:30,773 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:30,773 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,774 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf2 2024-11-14T09:29:30,774 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:30,775 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:30,775 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,776 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,777 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,778 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,778 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,779 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T09:29:30,781 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,781 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened e5dc7f00d707752df5f2aa388572824b; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67370525, jitterRate=0.0038990527391433716}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T09:29:30,782 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:30,782 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e5dc7f00d707752df5f2aa388572824b: Running coprocessor pre-open hook at 1731576570764Writing region info on filesystem at 1731576570764Initializing all the Stores at 1731576570765 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576570765Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576570765Cleaning up temporary data from old regions at 1731576570778 (+13 ms)Running coprocessor post-open hooks at 1731576570782 (+4 ms)Region opened successfully at 1731576570782 2024-11-14T09:29:30,783 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., pid=9, masterSystemTime=1731576570758 2024-11-14T09:29:30,786 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,786 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:30,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=OPEN, openSeqNum=9, regionLocation=83f56b55f2af,32943,1731576553877 2024-11-14T09:29:30,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,32943,1731576553877 because future has completed 2024-11-14T09:29:30,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-14T09:29:30,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,32943,1731576553877 in 185 msec 2024-11-14T09:29:30,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE in 591 msec 2024-11-14T09:29:30,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T09:29:30,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T09:29:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:46688 deadline: 1731576630818, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=83f56b55f2af port=32943 startCode=1731576553877. As of locationSeqNum=5. 2024-11-14T09:29:30,825 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=83f56b55f2af port=32943 startCode=1731576553877. As of locationSeqNum=5. 2024-11-14T09:29:30,825 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=83f56b55f2af port=32943 startCode=1731576553877. As of locationSeqNum=5. 2024-11-14T09:29:30,825 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,32943,1731576553877, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=83f56b55f2af port=32943 startCode=1731576553877. As of locationSeqNum=5. 2024-11-14T09:29:30,935 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:29:30,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:29:30,947 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e5dc7f00d707752df5f2aa388572824b 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-14T09:29:30,966 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf1/d98131a7985a4a1fa4a6a748116591a4 is 29, key is r1/cf1:/1731576570939/DeleteFamily/seqid=0 2024-11-14T09:29:30,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741888_1066 (size=4906) 2024-11-14T09:29:30,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741888_1066 (size=4906) 2024-11-14T09:29:30,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741888_1066 (size=4906) 2024-11-14T09:29:30,974 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf1/d98131a7985a4a1fa4a6a748116591a4 2024-11-14T09:29:30,980 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d98131a7985a4a1fa4a6a748116591a4 2024-11-14T09:29:31,002 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf2/0af87265c7ca4a9aa4cf92604787959a is 29, key is r1/cf2:/1731576570939/DeleteFamily/seqid=0 2024-11-14T09:29:31,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741889_1067 (size=4906) 2024-11-14T09:29:31,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741889_1067 (size=4906) 2024-11-14T09:29:31,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741889_1067 (size=4906) 2024-11-14T09:29:31,010 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf2/0af87265c7ca4a9aa4cf92604787959a 2024-11-14T09:29:31,015 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0af87265c7ca4a9aa4cf92604787959a 2024-11-14T09:29:31,016 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf1/d98131a7985a4a1fa4a6a748116591a4 as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/d98131a7985a4a1fa4a6a748116591a4 2024-11-14T09:29:31,023 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d98131a7985a4a1fa4a6a748116591a4 2024-11-14T09:29:31,023 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/d98131a7985a4a1fa4a6a748116591a4, entries=1, sequenceid=12, filesize=4.8 K 2024-11-14T09:29:31,025 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf2/0af87265c7ca4a9aa4cf92604787959a as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/0af87265c7ca4a9aa4cf92604787959a 2024-11-14T09:29:31,031 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0af87265c7ca4a9aa4cf92604787959a 2024-11-14T09:29:31,031 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/0af87265c7ca4a9aa4cf92604787959a, entries=1, sequenceid=12, filesize=4.8 K 2024-11-14T09:29:31,033 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for e5dc7f00d707752df5f2aa388572824b in 86ms, sequenceid=12, compaction requested=false 2024-11-14T09:29:31,033 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e5dc7f00d707752df5f2aa388572824b: 2024-11-14T09:29:31,035 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T09:29:31,037 DEBUG [Time-limited test {}] regionserver.HStore(1541): e5dc7f00d707752df5f2aa388572824b/cf1 is initiating major compaction (all files) 2024-11-14T09:29:31,037 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T09:29:31,037 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:31,037 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e5dc7f00d707752df5f2aa388572824b/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,037 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/a7dbe2683d5645cea14edbc20111c949, hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/d98131a7985a4a1fa4a6a748116591a4] into tmpdir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp, totalSize=9.5 K 2024-11-14T09:29:31,038 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a7dbe2683d5645cea14edbc20111c949, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731576570187 2024-11-14T09:29:31,039 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d98131a7985a4a1fa4a6a748116591a4, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-14T09:29:31,049 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): e5dc7f00d707752df5f2aa388572824b#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:29:31,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741890_1068 (size=4626) 2024-11-14T09:29:31,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741890_1068 (size=4626) 2024-11-14T09:29:31,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741890_1068 (size=4626) 2024-11-14T09:29:31,064 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf1/be4f3ad111414754a0e8c9c247c30289 as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/be4f3ad111414754a0e8c9c247c30289 2024-11-14T09:29:31,077 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in e5dc7f00d707752df5f2aa388572824b/cf1 of e5dc7f00d707752df5f2aa388572824b into be4f3ad111414754a0e8c9c247c30289(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:29:31,077 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e5dc7f00d707752df5f2aa388572824b: 2024-11-14T09:29:31,077 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-14T09:29:31,077 DEBUG [Time-limited test {}] regionserver.HStore(1541): e5dc7f00d707752df5f2aa388572824b/cf2 is initiating major compaction (all files) 2024-11-14T09:29:31,078 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T09:29:31,078 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:29:31,078 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e5dc7f00d707752df5f2aa388572824b/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,078 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/0af87265c7ca4a9aa4cf92604787959a] into tmpdir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp, totalSize=4.8 K 2024-11-14T09:29:31,078 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0af87265c7ca4a9aa4cf92604787959a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-14T09:29:31,084 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): e5dc7f00d707752df5f2aa388572824b#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:29:31,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741891_1069 (size=4592) 2024-11-14T09:29:31,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741891_1069 (size=4592) 2024-11-14T09:29:31,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741891_1069 (size=4592) 2024-11-14T09:29:31,103 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/.tmp/cf2/3ababe553d534f6fa515381917309356 as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/3ababe553d534f6fa515381917309356 2024-11-14T09:29:31,111 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in e5dc7f00d707752df5f2aa388572824b/cf2 of e5dc7f00d707752df5f2aa388572824b into 3ababe553d534f6fa515381917309356(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:29:31,111 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e5dc7f00d707752df5f2aa388572824b: 2024-11-14T09:29:31,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e5dc7f00d707752df5f2aa388572824b, source=83f56b55f2af,32943,1731576553877, destination=83f56b55f2af,35921,1731576554024, warming up region on 83f56b55f2af,35921,1731576554024 2024-11-14T09:29:31,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e5dc7f00d707752df5f2aa388572824b, source=83f56b55f2af,32943,1731576553877, destination=83f56b55f2af,35921,1731576554024, running balancer 2024-11-14T09:29:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE 2024-11-14T09:29:31,118 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE 2024-11-14T09:29:31,119 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=CLOSING, regionLocation=83f56b55f2af,32943,1731576553877 2024-11-14T09:29:31,119 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] regionserver.HRegion(7855): Warmup {ENCODED => e5dc7f00d707752df5f2aa388572824b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:31,121 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE because future has completed 2024-11-14T09:29:31,123 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:29:31,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,32943,1731576553877}] 2024-11-14T09:29:31,123 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf1 2024-11-14T09:29:31,123 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:31,131 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/a7dbe2683d5645cea14edbc20111c949 2024-11-14T09:29:31,136 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/be4f3ad111414754a0e8c9c247c30289 2024-11-14T09:29:31,141 INFO [StoreFileOpener-e5dc7f00d707752df5f2aa388572824b-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d98131a7985a4a1fa4a6a748116591a4 2024-11-14T09:29:31,141 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/d98131a7985a4a1fa4a6a748116591a4 2024-11-14T09:29:31,141 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:31,142 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,143 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf2 2024-11-14T09:29:31,143 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:31,149 INFO [StoreFileOpener-e5dc7f00d707752df5f2aa388572824b-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0af87265c7ca4a9aa4cf92604787959a 2024-11-14T09:29:31,150 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/0af87265c7ca4a9aa4cf92604787959a 2024-11-14T09:29:31,156 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/3ababe553d534f6fa515381917309356 2024-11-14T09:29:31,156 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] regionserver.HRegion(1722): Closing e5dc7f00d707752df5f2aa388572824b, disabling compactions & flushes 2024-11-14T09:29:31,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. after waiting 0 ms 2024-11-14T09:29:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35921 {}] regionserver.HRegion(1676): Region close journal for e5dc7f00d707752df5f2aa388572824b: Waiting for close lock at 1731576571156Disabling compacts and flushes for region at 1731576571156Disabling writes for close at 1731576571156Writing region close event to WAL at 1731576571161 (+5 ms)Closed at 1731576571161 2024-11-14T09:29:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-14T09:29:31,278 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,278 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:29:31,278 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing e5dc7f00d707752df5f2aa388572824b, disabling compactions & flushes 2024-11-14T09:29:31,278 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,278 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,278 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. after waiting 0 ms 2024-11-14T09:29:31,278 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,279 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/a7dbe2683d5645cea14edbc20111c949, hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/d98131a7985a4a1fa4a6a748116591a4] to archive 2024-11-14T09:29:31,283 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:29:31,287 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/a7dbe2683d5645cea14edbc20111c949 to hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/a7dbe2683d5645cea14edbc20111c949 2024-11-14T09:29:31,289 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/d98131a7985a4a1fa4a6a748116591a4 to hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/d98131a7985a4a1fa4a6a748116591a4 2024-11-14T09:29:31,302 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/0af87265c7ca4a9aa4cf92604787959a] to archive 2024-11-14T09:29:31,303 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:29:31,305 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/0af87265c7ca4a9aa4cf92604787959a to hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/0af87265c7ca4a9aa4cf92604787959a 2024-11-14T09:29:31,310 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-14T09:29:31,311 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,311 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for e5dc7f00d707752df5f2aa388572824b: Waiting for close lock at 1731576571278Running coprocessor pre-close hooks at 1731576571278Disabling compacts and flushes for region at 1731576571278Disabling writes for close at 1731576571278Writing region close event to WAL at 1731576571306 (+28 ms)Running coprocessor post-close hooks at 1731576571311 (+5 ms)Closed at 1731576571311 2024-11-14T09:29:31,311 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding e5dc7f00d707752df5f2aa388572824b move to 83f56b55f2af,35921,1731576554024 record at close sequenceid=12 2024-11-14T09:29:31,314 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,315 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=CLOSED 2024-11-14T09:29:31,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,32943,1731576553877 because future has completed 2024-11-14T09:29:31,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-14T09:29:31,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,32943,1731576553877 in 201 msec 2024-11-14T09:29:31,327 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE; state=CLOSED, location=83f56b55f2af,35921,1731576554024; forceNewPlan=false, retain=false 2024-11-14T09:29:31,477 INFO [83f56b55f2af:43303 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T09:29:31,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=OPENING, regionLocation=83f56b55f2af,35921,1731576554024 2024-11-14T09:29:31,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE because future has completed 2024-11-14T09:29:31,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,35921,1731576554024}] 2024-11-14T09:29:31,637 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,638 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => e5dc7f00d707752df5f2aa388572824b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:31,638 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,638 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:31,638 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,638 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,639 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,640 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf1 2024-11-14T09:29:31,640 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:31,647 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/be4f3ad111414754a0e8c9c247c30289 2024-11-14T09:29:31,647 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:31,648 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,649 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf2 2024-11-14T09:29:31,649 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:31,654 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/3ababe553d534f6fa515381917309356 2024-11-14T09:29:31,655 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:31,655 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,656 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,657 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,657 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,657 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,658 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T09:29:31,659 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,660 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened e5dc7f00d707752df5f2aa388572824b; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59711740, jitterRate=-0.11022573709487915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T09:29:31,660 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,661 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for e5dc7f00d707752df5f2aa388572824b: Running coprocessor pre-open hook at 1731576571638Writing region info on filesystem at 1731576571638Initializing all the Stores at 1731576571639 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576571639Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576571639Cleaning up temporary data from old regions at 1731576571657 (+18 ms)Running coprocessor post-open hooks at 1731576571660 (+3 ms)Region opened successfully at 1731576571661 (+1 ms) 2024-11-14T09:29:31,662 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., pid=12, masterSystemTime=1731576571633 2024-11-14T09:29:31,665 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,665 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,666 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=OPEN, openSeqNum=18, regionLocation=83f56b55f2af,35921,1731576554024 2024-11-14T09:29:31,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,35921,1731576554024 because future has completed 2024-11-14T09:29:31,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-14T09:29:31,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,35921,1731576554024 in 188 msec 2024-11-14T09:29:31,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, REOPEN/MOVE in 556 msec 2024-11-14T09:29:31,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T09:29:31,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T09:29:31,723 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 83f56b55f2af,35921,1731576554024: testing ***** 2024-11-14T09:29:31,723 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-14T09:29:31,725 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-14T09:29:31,726 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-14T09:29:31,731 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-14T09:29:31,732 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-14T09:29:31,742 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 441898032 }, "NonHeapMemoryUsage": { "committed": 171180032, "init": 7667712, "max": -1, "used": 168636736 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "83f56b55f2af", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 45, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 12, "ProcessCallTime_98th_percentile": 45, "ProcessCallTime_99th_percentile": 45, "ProcessCallTime_99.9th_percentile": 45, "ProcessCallTime_TimeRangeCount_0-1": 13, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 4, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 45, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 12, "TotalCallTime_98th_percentile": 45, "TotalCallTime_99th_percentile": 45, "TotalCallTime_99.9th_percentile": 45, "TotalCallTime_TimeRangeCount_0-1": 11, "TotalCallTime_TimeRangeCount_1-3": 14, "TotalCallTime_TimeRangeCount_3-10": 4, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5773 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "83f56b55f2af", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59004", "tag.serverName": "83f56b55f2af,35921,1731576554024", "tag.clusterId": "b68c35d9-ffc1-44fd-aa0b-01bef063b027", "tag.Context": "regionserver", "tag.Hostname": "83f56b55f2af", "regionCount": 2, "storeCount": 6, "hlogFileCount": 6, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 2275, "memStoreHeapSize": 5184, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731576554024, "averageRegionSize": 1137, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.8, "writeRequestRatePerSecond": 1.4, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 599040, "ByteBuffAllocatorHeapAllocationRatio": 0.35775729671400613, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 29, "totalRowActionRequestCount": 20, "readRequestCount": 11, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 9, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 17, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 1, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 5, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 16, "Put_min": 1, "Put_max": 3, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 3, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_0-1": 6, "Put_TimeRangeCount_1-3": 8, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 48, "FlushTime_max": 48, "FlushTime_mean": 48, "FlushTime_25th_percentile": 48, "FlushTime_median": 48, "FlushTime_75th_percentile": 48, "FlushTime_90th_percentile": 48, "FlushTime_95th_percentile": 48, "FlushTime_98th_percentile": 48, "FlushTime_99th_percentile": 48, "FlushTime_99.9th_percentile": 48, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 64, "ScanSize_max": 1112, "ScanSize_mean": 902, "ScanSize_25th_percentile": 308, "ScanSize_median": 576, "ScanSize_75th_percentile": 844, "ScanSize_90th_percentile": 1004, "ScanSize_95th_percentile": 1058, "ScanSize_98th_percentile": 1090, "ScanSize_99th_percentile": 1101, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_10-100": 1, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-14T09:29:31,746 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43303 {}] master.MasterRpcServices(700): 83f56b55f2af,35921,1731576554024 reported a fatal error: ***** ABORTING region server 83f56b55f2af,35921,1731576554024: testing ***** 2024-11-14T09:29:31,748 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '83f56b55f2af,35921,1731576554024' ***** 2024-11-14T09:29:31,748 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-14T09:29:31,749 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T09:29:31,749 INFO [RS:2;83f56b55f2af:35921 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-14T09:29:31,749 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T09:29:31,749 INFO [RS:2;83f56b55f2af:35921 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-14T09:29:31,749 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(3091): Received CLOSE for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,750 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(956): aborting server 83f56b55f2af,35921,1731576554024 2024-11-14T09:29:31,750 INFO [RS:2;83f56b55f2af:35921 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:29:31,750 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e5dc7f00d707752df5f2aa388572824b, disabling compactions & flushes 2024-11-14T09:29:31,750 INFO [RS:2;83f56b55f2af:35921 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;83f56b55f2af:35921. 2024-11-14T09:29:31,750 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32943 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:39510 deadline: 1731576631750, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=83f56b55f2af port=35921 startCode=1731576554024. As of locationSeqNum=12. 2024-11-14T09:29:31,750 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,750 DEBUG [RS:2;83f56b55f2af:35921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:29:31,750 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. after waiting 0 ms 2024-11-14T09:29:31,750 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,751 DEBUG [RS:2;83f56b55f2af:35921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:29:31,751 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,32943,1731576553877, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,32943,1731576553877, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=83f56b55f2af port=35921 startCode=1731576554024. As of locationSeqNum=12. 2024-11-14T09:29:31,751 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T09:29:31,751 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,32943,1731576553877, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=83f56b55f2af port=35921 startCode=1731576554024. As of locationSeqNum=12. 2024-11-14T09:29:31,751 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T09:29:31,751 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T09:29:31,751 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,32943,1731576553877, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=83f56b55f2af port=35921 startCode=1731576554024. As of locationSeqNum=12. 2024-11-14T09:29:31,751 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T09:29:31,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 83f56b55f2af,35921,1731576554024 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:31,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 83f56b55f2af,35921,1731576554024 aborting 2024-11-14T09:29:31,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 83f56b55f2af,35921,1731576554024 aborting 2024-11-14T09:29:31,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 from cache 2024-11-14T09:29:31,757 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T09:29:31,758 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:29:31,758 DEBUG [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, e5dc7f00d707752df5f2aa388572824b=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.} 2024-11-14T09:29:31,758 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:29:31,758 DEBUG [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:31,758 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:29:31,758 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:29:31,758 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:29:31,759 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,759 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e5dc7f00d707752df5f2aa388572824b: Waiting for close lock at 1731576571750Running coprocessor pre-close hooks at 1731576571750Disabling compacts and flushes for region at 1731576571750Disabling writes for close at 1731576571750Writing region close event to WAL at 1731576571758 (+8 ms)Running coprocessor post-close hooks at 1731576571758Closed at 1731576571759 (+1 ms) 2024-11-14T09:29:31,759 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:31,759 ERROR [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-14T09:29:31,759 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T09:29:31,760 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:29:31,760 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731576571757Running coprocessor pre-close hooks at 1731576571757Disabling compacts and flushes for region at 1731576571758 (+1 ms)Disabling writes for close at 1731576571758Writing region close event to WAL at 1731576571759 (+1 ms)Running coprocessor post-close hooks at 1731576571759Closed at 1731576571760 (+1 ms) 2024-11-14T09:29:31,760 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T09:29:31,773 INFO [regionserver/83f56b55f2af:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:29:31,865 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:29:31,867 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1] 2024-11-14T09:29:31,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 83f56b55f2af,35921,1731576554024 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:31,868 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 83f56b55f2af,35921,1731576554024 aborting 2024-11-14T09:29:31,868 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 83f56b55f2af,35921,1731576554024 aborting 2024-11-14T09:29:31,868 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 from cache 2024-11-14T09:29:31,958 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(976): stopping server 83f56b55f2af,35921,1731576554024; all regions closed. 2024-11-14T09:29:31,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741836_1012 (size=6845) 2024-11-14T09:29:31,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741836_1012 (size=6845) 2024-11-14T09:29:31,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741836_1012 (size=6845) 2024-11-14T09:29:31,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741835_1011 (size=2142) 2024-11-14T09:29:31,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741835_1011 (size=2142) 2024-11-14T09:29:31,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741835_1011 (size=2142) 2024-11-14T09:29:31,966 DEBUG [RS:2;83f56b55f2af:35921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:29:31,966 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:29:31,966 INFO [RS:2;83f56b55f2af:35921 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:29:31,967 INFO [RS:2;83f56b55f2af:35921 {}] hbase.ChoreService(370): Chore service for: regionserver/83f56b55f2af:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T09:29:31,967 INFO [regionserver/83f56b55f2af:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:29:31,967 INFO [RS:2;83f56b55f2af:35921 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:29:31,968 INFO [RS:2;83f56b55f2af:35921 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35921 2024-11-14T09:29:31,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/83f56b55f2af,35921,1731576554024 2024-11-14T09:29:31,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:29:31,975 INFO [RS:2;83f56b55f2af:35921 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:29:31,977 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [83f56b55f2af,35921,1731576554024] 2024-11-14T09:29:31,979 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/83f56b55f2af,35921,1731576554024 already deleted, retry=false 2024-11-14T09:29:31,979 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 83f56b55f2af,35921,1731576554024 on 83f56b55f2af,43303,1731576553096 2024-11-14T09:29:31,984 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 83f56b55f2af,35921,1731576554024, splitWal=true, meta=true 2024-11-14T09:29:31,986 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 83f56b55f2af,35921,1731576554024 (carryingMeta=true) 83f56b55f2af,35921,1731576554024/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@66a91a5a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T09:29:31,987 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 83f56b55f2af,35921,1731576554024, splitWal=true, meta=true 2024-11-14T09:29:31,988 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 83f56b55f2af,35921,1731576554024, splitWal=true, meta=true, isMeta: true 2024-11-14T09:29:31,990 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting 2024-11-14T09:29:31,991 INFO [PEWorker-1 {}] master.SplitWALManager(105): 83f56b55f2af,35921,1731576554024 WAL count=1, meta=true 2024-11-14T09:29:31,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta}] 2024-11-14T09:29:32,000 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=83f56b55f2af,32943,1731576553877 2024-11-14T09:29:32,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta, worker=83f56b55f2af,32943,1731576553877}] 2024-11-14T09:29:32,075 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:29:32,076 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1] 2024-11-14T09:29:32,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:29:32,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35921-0x10115cf45f40003, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:29:32,078 INFO [RS:2;83f56b55f2af:35921 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:29:32,078 INFO [RS:2;83f56b55f2af:35921 {}] regionserver.HRegionServer(1031): Exiting; stopping=83f56b55f2af,35921,1731576554024; zookeeper connection closed. 2024-11-14T09:29:32,078 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 83f56b55f2af:35921 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 83f56b55f2af/172.17.0.2:35921 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:32,079 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1606448e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1606448e 2024-11-14T09:29:32,079 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1, error=java.net.ConnectException: Call to address=83f56b55f2af:35921 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 83f56b55f2af/172.17.0.2:35921 2024-11-14T09:29:32,079 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-14T09:29:32,079 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 from cache 2024-11-14T09:29:32,080 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 83f56b55f2af:35921 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 83f56b55f2af/172.17.0.2:35921 2024-11-14T09:29:32,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32943 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-14T09:29:32,184 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta, size=6.7 K (6845bytes) 2024-11-14T09:29:32,184 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta 2024-11-14T09:29:32,184 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta after 0ms 2024-11-14T09:29:32,187 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta took 4ms 2024-11-14T09:29:32,194 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-14T09:29:32,194 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta so closing down 2024-11-14T09:29:32,195 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:32,196 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta.temp 2024-11-14T09:29:32,197 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta.temp 2024-11-14T09:29:32,198 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:32,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741892_1070 (size=6604) 2024-11-14T09:29:32,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741892_1070 (size=6604) 2024-11-14T09:29:32,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741892_1070 (size=6604) 2024-11-14T09:29:32,207 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:32,208 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta.temp to hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T09:29:32,208 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 21 ms; skipped=1; WAL=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-11-14T09:29:32,209 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta, journal: Splitting hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta, size=6.7 K (6845bytes) at 1731576572184Finishing writing output for hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta so closing down at 1731576572194 (+10 ms)Creating recovered edits writer path=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta.temp at 1731576572197 (+3 ms)3 split writer threads finished at 1731576572198 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731576572207 (+9 ms)Rename recovered edits hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta.temp to hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1731576572208 (+1 ms)Processed 16 edits across 1 Regions in 21 ms; skipped=1; WAL=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1731576572208 2024-11-14T09:29:32,209 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta 2024-11-14T09:29:32,210 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-14T09:29:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-14T09:29:32,216 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta to hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/oldWALs 2024-11-14T09:29:32,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-14T09:29:32,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta, worker=83f56b55f2af,32943,1731576553877 in 214 msec 2024-11-14T09:29:32,220 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=83f56b55f2af,32943,1731576553877 2024-11-14T09:29:32,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T09:29:32,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 83f56b55f2af%2C35921%2C1731576554024.meta.1731576555866.meta, worker=83f56b55f2af,32943,1731576553877 in 228 msec 2024-11-14T09:29:32,225 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting dir is empty, no logs to split. 2024-11-14T09:29:32,225 INFO [PEWorker-1 {}] master.SplitWALManager(105): 83f56b55f2af,35921,1731576554024 WAL count=0, meta=true 2024-11-14T09:29:32,225 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 83f56b55f2af,35921,1731576554024 WAL splitting is done? wals=0, meta=true 2024-11-14T09:29:32,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T09:29:32,227 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:29:32,229 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T09:29:32,379 DEBUG [83f56b55f2af:43303 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-14T09:29:32,379 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(204): Hosts are {83f56b55f2af=0} racks are {/default-rack=0} 2024-11-14T09:29:32,380 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T09:29:32,380 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T09:29:32,380 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T09:29:32,380 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T09:29:32,380 INFO [83f56b55f2af:43303 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T09:29:32,380 INFO [83f56b55f2af:43303 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T09:29:32,380 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T09:29:32,380 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=83f56b55f2af,43391,1731576553984 2024-11-14T09:29:32,381 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 83f56b55f2af,43391,1731576553984, state=OPENING 2024-11-14T09:29:32,385 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:29:32,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:29:32,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:29:32,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:29:32,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:32,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:32,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:32,385 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:29:32,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=83f56b55f2af,43391,1731576553984}] 2024-11-14T09:29:32,386 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1] 2024-11-14T09:29:32,387 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 83f56b55f2af:35921 this server is in the failed servers list 2024-11-14T09:29:32,387 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=83f56b55f2af:35921 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 83f56b55f2af:35921 2024-11-14T09:29:32,387 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 83f56b55f2af:35921 2024-11-14T09:29:32,387 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 from cache 2024-11-14T09:29:32,538 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T09:29:32,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44963, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T09:29:32,545 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T09:29:32,545 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:32,545 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T09:29:32,547 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=83f56b55f2af%2C43391%2C1731576553984.meta, suffix=.meta, logDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,43391,1731576553984, archiveDir=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/oldWALs, maxLogs=32 2024-11-14T09:29:32,561 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,43391,1731576553984/83f56b55f2af%2C43391%2C1731576553984.meta.1731576572547.meta, exclude list is [], retry=0 2024-11-14T09:29:32,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:32,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:32,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:32,568 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,43391,1731576553984/83f56b55f2af%2C43391%2C1731576553984.meta.1731576572547.meta 2024-11-14T09:29:32,568 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:32,569 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:32,569 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T09:29:32,569 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T09:29:32,569 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T09:29:32,569 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T09:29:32,569 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:32,569 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T09:29:32,569 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T09:29:32,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T09:29:32,572 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:29:32,572 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:32,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:32,573 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T09:29:32,573 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:29:32,573 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:32,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:32,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T09:29:32,574 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:29:32,574 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:32,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:32,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T09:29:32,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:29:32,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:32,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:29:32,576 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:29:32,577 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740 2024-11-14T09:29:32,579 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740 2024-11-14T09:29:32,579 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T09:29:32,583 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T09:29:32,583 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-14T09:29:32,605 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/info/0febdb15098f4c2c81966872b82db565 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b./info:regioninfo/1731576571665/Put/seqid=0 2024-11-14T09:29:32,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741894_1072 (size=11177) 2024-11-14T09:29:32,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741894_1072 (size=11177) 2024-11-14T09:29:32,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741894_1072 (size=11177) 2024-11-14T09:29:32,613 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/info/0febdb15098f4c2c81966872b82db565 2024-11-14T09:29:32,641 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/ns/7ae01e0f74924711bd745d962c0a130a is 43, key is default/ns:d/1731576556094/Put/seqid=0 2024-11-14T09:29:32,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741895_1073 (size=5153) 2024-11-14T09:29:32,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741895_1073 (size=5153) 2024-11-14T09:29:32,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741895_1073 (size=5153) 2024-11-14T09:29:32,649 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/ns/7ae01e0f74924711bd745d962c0a130a 2024-11-14T09:29:32,670 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/table/71e2bceefeb84067b7e2948b2b66379e is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731576569941/Put/seqid=0 2024-11-14T09:29:32,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741896_1074 (size=5431) 2024-11-14T09:29:32,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741896_1074 (size=5431) 2024-11-14T09:29:32,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741896_1074 (size=5431) 2024-11-14T09:29:32,678 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/table/71e2bceefeb84067b7e2948b2b66379e 2024-11-14T09:29:32,685 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/info/0febdb15098f4c2c81966872b82db565 as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/info/0febdb15098f4c2c81966872b82db565 2024-11-14T09:29:32,690 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/info/0febdb15098f4c2c81966872b82db565, entries=36, sequenceid=18, filesize=10.9 K 2024-11-14T09:29:32,691 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/ns/7ae01e0f74924711bd745d962c0a130a as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/ns/7ae01e0f74924711bd745d962c0a130a 2024-11-14T09:29:32,696 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/ns/7ae01e0f74924711bd745d962c0a130a, entries=2, sequenceid=18, filesize=5.0 K 2024-11-14T09:29:32,697 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/table/71e2bceefeb84067b7e2948b2b66379e as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/table/71e2bceefeb84067b7e2948b2b66379e 2024-11-14T09:29:32,702 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/table/71e2bceefeb84067b7e2948b2b66379e, entries=2, sequenceid=18, filesize=5.3 K 2024-11-14T09:29:32,702 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 119ms, sequenceid=18, compaction requested=false; wal=null 2024-11-14T09:29:32,703 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T09:29:32,704 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:29:32,704 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:29:32,705 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T09:29:32,706 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:29:32,708 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-14T09:29:32,709 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73270687, jitterRate=0.09181831777095795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T09:29:32,709 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T09:29:32,710 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731576572570Writing region info on filesystem at 1731576572570Initializing all the Stores at 1731576572570Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576572570Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576572571 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576572571Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731576572571Obtaining lock to block concurrent updates at 1731576572583 (+12 ms)Preparing flush snapshotting stores in 1588230740 at 1731576572583Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1731576572583Flushing stores of hbase:meta,,1.1588230740 at 1731576572583Flushing 1588230740/info: creating writer at 1731576572583Flushing 1588230740/info: appending metadata at 1731576572604 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731576572604Flushing 1588230740/ns: creating writer at 1731576572619 (+15 ms)Flushing 1588230740/ns: appending metadata at 1731576572640 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731576572640Flushing 1588230740/table: creating writer at 1731576572654 (+14 ms)Flushing 1588230740/table: appending metadata at 1731576572670 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731576572670Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c336f63: reopening flushed file at 1731576572684 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16ade76a: reopening flushed file at 1731576572690 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41327d2e: reopening flushed file at 1731576572696 (+6 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 119ms, sequenceid=18, compaction requested=false; wal=null at 1731576572702 (+6 ms)Cleaning up temporary data from old regions at 1731576572704 (+2 ms)Running coprocessor post-open hooks at 1731576572709 (+5 ms)Region opened successfully at 1731576572710 (+1 ms) 2024-11-14T09:29:32,711 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1731576572538 2024-11-14T09:29:32,714 DEBUG [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T09:29:32,714 INFO [RS_OPEN_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T09:29:32,714 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=83f56b55f2af,43391,1731576553984 2024-11-14T09:29:32,715 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 83f56b55f2af,43391,1731576553984, state=OPEN 2024-11-14T09:29:32,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:29:32,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:29:32,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:29:32,717 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=83f56b55f2af,43391,1731576553984 2024-11-14T09:29:32,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:32,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:32,718 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:29:32,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-14T09:29:32,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=83f56b55f2af,43391,1731576553984 in 332 msec 2024-11-14T09:29:32,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-14T09:29:32,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 495 msec 2024-11-14T09:29:32,724 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 83f56b55f2af,35921,1731576554024 had 2 regions 2024-11-14T09:29:32,725 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 83f56b55f2af,35921,1731576554024, splitWal=true, meta=true, isMeta: false 2024-11-14T09:29:32,727 INFO [PEWorker-2 {}] master.SplitWALManager(105): 83f56b55f2af,35921,1731576554024 WAL count=1, meta=false 2024-11-14T09:29:32,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 83f56b55f2af%2C35921%2C1731576554024.1731576555517}] 2024-11-14T09:29:32,729 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=83f56b55f2af,43391,1731576553984 2024-11-14T09:29:32,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 83f56b55f2af%2C35921%2C1731576554024.1731576555517, worker=83f56b55f2af,43391,1731576553984}] 2024-11-14T09:29:32,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43391 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-14T09:29:32,895 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:29:32,896 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,43391,1731576553984, seqNum=-1] 2024-11-14T09:29:32,896 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:29:32,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:29:32,903 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517, size=2.1 K (2142bytes) 2024-11-14T09:29:32,903 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517 2024-11-14T09:29:32,903 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517 after 0ms 2024-11-14T09:29:32,906 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517 took 4ms 2024-11-14T09:29:32,909 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for e5dc7f00d707752df5f2aa388572824b: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-14T09:29:32,909 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517 so closing down 2024-11-14T09:29:32,909 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:32,909 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:32,909 INFO [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-14T09:29:32,910 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517, journal: Splitting hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517, size=2.1 K (2142bytes) at 1731576572903Finishing writing output for hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517 so closing down at 1731576572909 (+6 ms)3 split writer threads finished at 1731576572909Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1731576572909 2024-11-14T09:29:32,910 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517 2024-11-14T09:29:32,910 DEBUG [RS_LOG_REPLAY_OPS-regionserver/83f56b55f2af:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-14T09:29:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43303 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-14T09:29:32,914 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting/83f56b55f2af%2C35921%2C1731576554024.1731576555517 to hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/oldWALs 2024-11-14T09:29:32,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-14T09:29:32,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 83f56b55f2af%2C35921%2C1731576554024.1731576555517, worker=83f56b55f2af,43391,1731576553984 in 184 msec 2024-11-14T09:29:32,917 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=83f56b55f2af,43391,1731576553984 2024-11-14T09:29:32,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-14T09:29:32,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 83f56b55f2af%2C35921%2C1731576554024.1731576555517, worker=83f56b55f2af,43391,1731576553984 in 191 msec 2024-11-14T09:29:32,921 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/WALs/83f56b55f2af,35921,1731576554024-splitting dir is empty, no logs to split. 2024-11-14T09:29:32,922 INFO [PEWorker-2 {}] master.SplitWALManager(105): 83f56b55f2af,35921,1731576554024 WAL count=0, meta=false 2024-11-14T09:29:32,922 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 83f56b55f2af,35921,1731576554024 WAL splitting is done? wals=0, meta=false 2024-11-14T09:29:32,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, ASSIGN}] 2024-11-14T09:29:32,925 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, ASSIGN 2024-11-14T09:29:32,926 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T09:29:33,015 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 83f56b55f2af:35921 this server is in the failed servers list 2024-11-14T09:29:33,015 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=83f56b55f2af:35921 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 83f56b55f2af:35921 2024-11-14T09:29:33,016 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 83f56b55f2af:35921 2024-11-14T09:29:33,016 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=12 from cache 2024-11-14T09:29:33,076 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(204): Hosts are {83f56b55f2af=0} racks are {/default-rack=0} 2024-11-14T09:29:33,076 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T09:29:33,076 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T09:29:33,076 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T09:29:33,076 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T09:29:33,076 INFO [83f56b55f2af:43303 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T09:29:33,076 INFO [83f56b55f2af:43303 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T09:29:33,076 DEBUG [83f56b55f2af:43303 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T09:29:33,077 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=OPENING, regionLocation=83f56b55f2af,32943,1731576553877 2024-11-14T09:29:33,078 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 83f56b55f2af:35921 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 83f56b55f2af/172.17.0.2:35921 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:33,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1, error=java.net.ConnectException: Call to address=83f56b55f2af:35921 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 83f56b55f2af/172.17.0.2:35921 2024-11-14T09:29:33,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-14T09:29:33,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,35921,1731576554024, seqNum=-1 from cache 2024-11-14T09:29:33,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 83f56b55f2af:35921 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 83f56b55f2af/172.17.0.2:35921 2024-11-14T09:29:33,185 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:29:33,185 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=83f56b55f2af,43391,1731576553984, seqNum=-1] 2024-11-14T09:29:33,186 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:29:33,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54749, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:29:33,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, ASSIGN because future has completed 2024-11-14T09:29:33,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,32943,1731576553877}] 2024-11-14T09:29:33,233 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=18] 2024-11-14T09:29:33,233 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to 83f56b55f2af:35921 this server is in the failed servers list 2024-11-14T09:29:33,234 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=83f56b55f2af:35921 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 83f56b55f2af:35921 2024-11-14T09:29:33,234 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 83f56b55f2af:35921 2024-11-14T09:29:33,234 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,35921,1731576554024, seqNum=18 from cache 2024-11-14T09:29:33,346 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:33,346 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => e5dc7f00d707752df5f2aa388572824b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:33,346 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,346 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:33,347 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,347 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,348 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,349 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf1 2024-11-14T09:29:33,349 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:33,355 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf1/be4f3ad111414754a0e8c9c247c30289 2024-11-14T09:29:33,356 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:33,356 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,357 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5dc7f00d707752df5f2aa388572824b columnFamilyName cf2 2024-11-14T09:29:33,357 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:33,363 DEBUG [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/cf2/3ababe553d534f6fa515381917309356 2024-11-14T09:29:33,363 INFO [StoreOpener-e5dc7f00d707752df5f2aa388572824b-1 {}] regionserver.HStore(327): Store=e5dc7f00d707752df5f2aa388572824b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:33,363 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,364 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,365 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,366 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,366 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,366 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T09:29:33,368 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,369 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened e5dc7f00d707752df5f2aa388572824b; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65575377, jitterRate=-0.02285073697566986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T09:29:33,369 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:33,369 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for e5dc7f00d707752df5f2aa388572824b: Running coprocessor pre-open hook at 1731576573347Writing region info on filesystem at 1731576573347Initializing all the Stores at 1731576573348 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576573348Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576573348Cleaning up temporary data from old regions at 1731576573366 (+18 ms)Running coprocessor post-open hooks at 1731576573369 (+3 ms)Region opened successfully at 1731576573369 2024-11-14T09:29:33,370 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., pid=21, masterSystemTime=1731576573342 2024-11-14T09:29:33,372 DEBUG [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:33,373 INFO [RS_OPEN_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:33,373 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e5dc7f00d707752df5f2aa388572824b, regionState=OPEN, openSeqNum=18, regionLocation=83f56b55f2af,32943,1731576553877 2024-11-14T09:29:33,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,32943,1731576553877 because future has completed 2024-11-14T09:29:33,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-14T09:29:33,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure e5dc7f00d707752df5f2aa388572824b, server=83f56b55f2af,32943,1731576553877 in 187 msec 2024-11-14T09:29:33,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-14T09:29:33,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e5dc7f00d707752df5f2aa388572824b, ASSIGN in 455 msec 2024-11-14T09:29:33,382 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 83f56b55f2af,35921,1731576554024 after splitting done 2024-11-14T09:29:33,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 83f56b55f2af,35921,1731576554024, splitWal=true, meta=true in 1.4020 sec 2024-11-14T09:29:33,473 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T09:29:33,549 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b., hostname=83f56b55f2af,32943,1731576553877, seqNum=18] 2024-11-14T09:29:33,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 409), OpenFileDescriptor=1048 (was 1021) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=262 (was 258) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6753 (was 6827) 2024-11-14T09:29:33,567 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-14T09:29:33,580 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=262, ProcessCount=11, AvailableMemoryMB=6752 2024-11-14T09:29:33,580 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-14T09:29:33,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-14T09:29:33,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-14T09:29:33,583 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-14T09:29:33,583 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-14T09:29:33,583 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-14T09:29:33,583 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-14T09:29:33,597 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:33,599 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:33,599 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:33,601 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45250555, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/hregion-45250555, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:33,614 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45250555/hregion-45250555.1731576573602, exclude list is [], retry=0 2024-11-14T09:29:33,617 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:33,617 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:33,617 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:33,619 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45250555/hregion-45250555.1731576573602 2024-11-14T09:29:33,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:33,620 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 30459879474abb0f5bba1f21ffa509ca, NAME => 'testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/hbase 2024-11-14T09:29:33,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741898_1076 (size=67) 2024-11-14T09:29:33,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741898_1076 (size=67) 2024-11-14T09:29:33,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741898_1076 (size=67) 2024-11-14T09:29:33,630 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:33,631 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,633 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30459879474abb0f5bba1f21ffa509ca columnFamilyName a 2024-11-14T09:29:33,633 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:33,633 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(327): Store=30459879474abb0f5bba1f21ffa509ca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:33,633 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,635 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30459879474abb0f5bba1f21ffa509ca columnFamilyName b 2024-11-14T09:29:33,635 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:33,635 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(327): Store=30459879474abb0f5bba1f21ffa509ca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:33,635 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,636 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30459879474abb0f5bba1f21ffa509ca columnFamilyName c 2024-11-14T09:29:33,637 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:33,637 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(327): Store=30459879474abb0f5bba1f21ffa509ca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:33,637 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,638 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,638 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,639 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,639 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,640 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:33,641 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,643 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:33,643 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 30459879474abb0f5bba1f21ffa509ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72317671, jitterRate=0.07761727273464203}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:33,644 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 30459879474abb0f5bba1f21ffa509ca: Writing region info on filesystem at 1731576573630Initializing all the Stores at 1731576573631 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576573631Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576573631Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576573631Cleaning up temporary data from old regions at 1731576573639 (+8 ms)Region opened successfully at 1731576573644 (+5 ms) 2024-11-14T09:29:33,644 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 30459879474abb0f5bba1f21ffa509ca, disabling compactions & flushes 2024-11-14T09:29:33,644 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. 2024-11-14T09:29:33,644 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. 2024-11-14T09:29:33,644 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. after waiting 0 ms 2024-11-14T09:29:33,644 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. 2024-11-14T09:29:33,644 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. 2024-11-14T09:29:33,645 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 30459879474abb0f5bba1f21ffa509ca: Waiting for close lock at 1731576573644Disabling compacts and flushes for region at 1731576573644Disabling writes for close at 1731576573644Writing region close event to WAL at 1731576573644Closed at 1731576573644 2024-11-14T09:29:33,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741897_1075 (size=93) 2024-11-14T09:29:33,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741897_1075 (size=93) 2024-11-14T09:29:33,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741897_1075 (size=93) 2024-11-14T09:29:33,650 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:33,651 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45250555:(num 1731576573602) 2024-11-14T09:29:33,651 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:33,652 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:33,666 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653, exclude list is [], retry=0 2024-11-14T09:29:33,668 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:33,669 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:33,669 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:33,671 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653 2024-11-14T09:29:33,671 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:33,671 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 30459879474abb0f5bba1f21ffa509ca, NAME => 'testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:33,672 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:33,672 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,672 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,673 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,674 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30459879474abb0f5bba1f21ffa509ca columnFamilyName a 2024-11-14T09:29:33,674 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:33,675 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(327): Store=30459879474abb0f5bba1f21ffa509ca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:33,675 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,676 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30459879474abb0f5bba1f21ffa509ca columnFamilyName b 2024-11-14T09:29:33,676 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:33,676 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(327): Store=30459879474abb0f5bba1f21ffa509ca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:33,676 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,677 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30459879474abb0f5bba1f21ffa509ca columnFamilyName c 2024-11-14T09:29:33,677 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:33,677 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(327): Store=30459879474abb0f5bba1f21ffa509ca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:33,677 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,678 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,679 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,680 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,680 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,681 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:33,682 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:33,683 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 30459879474abb0f5bba1f21ffa509ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59228923, jitterRate=-0.1174202710390091}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:33,683 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 30459879474abb0f5bba1f21ffa509ca: Writing region info on filesystem at 1731576573672Initializing all the Stores at 1731576573673 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576573673Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576573673Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576573673Cleaning up temporary data from old regions at 1731576573680 (+7 ms)Region opened successfully at 1731576573683 (+3 ms) 2024-11-14T09:29:33,713 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 30459879474abb0f5bba1f21ffa509ca 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-14T09:29:33,730 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/a/7edabd344abb4ed390efcadf0f695478 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731576573683/Put/seqid=0 2024-11-14T09:29:33,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741900_1078 (size=5958) 2024-11-14T09:29:33,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741900_1078 (size=5958) 2024-11-14T09:29:33,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741900_1078 (size=5958) 2024-11-14T09:29:33,743 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/a/7edabd344abb4ed390efcadf0f695478 2024-11-14T09:29:33,766 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/b/ab4cd1f5526947dca708c50962902d10 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731576573693/Put/seqid=0 2024-11-14T09:29:33,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741901_1079 (size=5958) 2024-11-14T09:29:33,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741901_1079 (size=5958) 2024-11-14T09:29:33,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741901_1079 (size=5958) 2024-11-14T09:29:33,780 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/b/ab4cd1f5526947dca708c50962902d10 2024-11-14T09:29:33,809 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/c/be5019a3420340798e49c7a77565c6f6 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731576573705/Put/seqid=0 2024-11-14T09:29:33,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741902_1080 (size=5958) 2024-11-14T09:29:33,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741902_1080 (size=5958) 2024-11-14T09:29:33,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741902_1080 (size=5958) 2024-11-14T09:29:34,221 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/c/be5019a3420340798e49c7a77565c6f6 2024-11-14T09:29:34,227 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/a/7edabd344abb4ed390efcadf0f695478 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/a/7edabd344abb4ed390efcadf0f695478 2024-11-14T09:29:34,232 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/a/7edabd344abb4ed390efcadf0f695478, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T09:29:34,233 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/b/ab4cd1f5526947dca708c50962902d10 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/b/ab4cd1f5526947dca708c50962902d10 2024-11-14T09:29:34,237 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/b/ab4cd1f5526947dca708c50962902d10, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T09:29:34,238 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/c/be5019a3420340798e49c7a77565c6f6 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/c/be5019a3420340798e49c7a77565c6f6 2024-11-14T09:29:34,243 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/c/be5019a3420340798e49c7a77565c6f6, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T09:29:34,244 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 30459879474abb0f5bba1f21ffa509ca in 531ms, sequenceid=33, compaction requested=false 2024-11-14T09:29:34,244 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 30459879474abb0f5bba1f21ffa509ca: 2024-11-14T09:29:34,244 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 30459879474abb0f5bba1f21ffa509ca, disabling compactions & flushes 2024-11-14T09:29:34,244 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. 2024-11-14T09:29:34,244 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. 2024-11-14T09:29:34,244 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. after waiting 0 ms 2024-11-14T09:29:34,244 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. 2024-11-14T09:29:34,245 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. 2024-11-14T09:29:34,246 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 30459879474abb0f5bba1f21ffa509ca: Waiting for close lock at 1731576574244Disabling compacts and flushes for region at 1731576574244Disabling writes for close at 1731576574244Writing region close event to WAL at 1731576574245 (+1 ms)Closed at 1731576574245 2024-11-14T09:29:34,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741899_1077 (size=5937) 2024-11-14T09:29:34,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741899_1077 (size=5937) 2024-11-14T09:29:34,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741899_1077 (size=5937) 2024-11-14T09:29:34,253 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/b/ab4cd1f5526947dca708c50962902d10 to hdfs://localhost:34615/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/b/ab4cd1f5526947dca708c50962902d10 2024-11-14T09:29:34,269 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653, size=5.8 K (5937bytes) 2024-11-14T09:29:34,269 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653 2024-11-14T09:29:34,269 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653 after 0ms 2024-11-14T09:29:34,271 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653 took 2ms 2024-11-14T09:29:34,273 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653 so closing down 2024-11-14T09:29:34,273 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:34,274 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731576573653.temp 2024-11-14T09:29:34,275 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000003-wal.1731576573653.temp 2024-11-14T09:29:34,276 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:34,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741903_1081 (size=5223) 2024-11-14T09:29:34,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741903_1081 (size=5223) 2024-11-14T09:29:34,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741903_1081 (size=5223) 2024-11-14T09:29:34,285 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000003-wal.1731576573653.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:34,287 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000003-wal.1731576573653.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000032 2024-11-14T09:29:34,287 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-14T09:29:34,287 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653, journal: Splitting hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653, size=5.8 K (5937bytes) at 1731576574269Finishing writing output for hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653 so closing down at 1731576574273 (+4 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000003-wal.1731576573653.temp at 1731576574275 (+2 ms)3 split writer threads finished at 1731576574276 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000003-wal.1731576573653.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731576574285 (+9 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000003-wal.1731576573653.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000032 at 1731576574287 (+2 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1731576574287 2024-11-14T09:29:34,288 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576573653 to hdfs://localhost:34615/hbase/oldWALs/wal.1731576573653 2024-11-14T09:29:34,289 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000032 2024-11-14T09:29:34,289 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:34,291 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:34,304 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576574291, exclude list is [], retry=0 2024-11-14T09:29:34,306 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:34,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:34,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:34,309 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576574291 2024-11-14T09:29:34,309 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:34,309 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 30459879474abb0f5bba1f21ffa509ca, NAME => 'testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:34,309 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:34,309 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,309 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,311 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,311 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30459879474abb0f5bba1f21ffa509ca columnFamilyName a 2024-11-14T09:29:34,312 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:34,318 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/a/7edabd344abb4ed390efcadf0f695478 2024-11-14T09:29:34,318 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(327): Store=30459879474abb0f5bba1f21ffa509ca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:34,318 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,320 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30459879474abb0f5bba1f21ffa509ca columnFamilyName b 2024-11-14T09:29:34,321 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:34,321 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(327): Store=30459879474abb0f5bba1f21ffa509ca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:34,321 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,322 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30459879474abb0f5bba1f21ffa509ca columnFamilyName c 2024-11-14T09:29:34,322 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:34,329 DEBUG [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/c/be5019a3420340798e49c7a77565c6f6 2024-11-14T09:29:34,329 INFO [StoreOpener-30459879474abb0f5bba1f21ffa509ca-1 {}] regionserver.HStore(327): Store=30459879474abb0f5bba1f21ffa509ca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:34,329 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,330 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,331 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,332 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000032 2024-11-14T09:29:34,335 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000032 2024-11-14T09:29:34,336 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 30459879474abb0f5bba1f21ffa509ca 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-14T09:29:34,357 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/b/6695c74058104c7e991a9362261b4750 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731576573693/Put/seqid=0 2024-11-14T09:29:34,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741905_1083 (size=5958) 2024-11-14T09:29:34,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741905_1083 (size=5958) 2024-11-14T09:29:34,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741905_1083 (size=5958) 2024-11-14T09:29:34,364 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/b/6695c74058104c7e991a9362261b4750 2024-11-14T09:29:34,371 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/.tmp/b/6695c74058104c7e991a9362261b4750 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/b/6695c74058104c7e991a9362261b4750 2024-11-14T09:29:34,376 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/b/6695c74058104c7e991a9362261b4750, entries=10, sequenceid=32, filesize=5.8 K 2024-11-14T09:29:34,376 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 30459879474abb0f5bba1f21ffa509ca in 41ms, sequenceid=32, compaction requested=false; wal=null 2024-11-14T09:29:34,377 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/0000000000000000032 2024-11-14T09:29:34,378 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,378 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,379 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:34,380 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 30459879474abb0f5bba1f21ffa509ca 2024-11-14T09:29:34,382 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/30459879474abb0f5bba1f21ffa509ca/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-14T09:29:34,383 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 30459879474abb0f5bba1f21ffa509ca; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67105337, jitterRate=-5.2556395530700684E-5}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:34,384 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 30459879474abb0f5bba1f21ffa509ca: Writing region info on filesystem at 1731576574309Initializing all the Stores at 1731576574310 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576574310Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576574310Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576574310Obtaining lock to block concurrent updates at 1731576574336 (+26 ms)Preparing flush snapshotting stores in 30459879474abb0f5bba1f21ffa509ca at 1731576574336Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731576574336Flushing stores of testReplayEditsWrittenViaHRegion,,1731576573597.30459879474abb0f5bba1f21ffa509ca. at 1731576574336Flushing 30459879474abb0f5bba1f21ffa509ca/b: creating writer at 1731576574336Flushing 30459879474abb0f5bba1f21ffa509ca/b: appending metadata at 1731576574356 (+20 ms)Flushing 30459879474abb0f5bba1f21ffa509ca/b: closing flushed file at 1731576574356Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@140da781: reopening flushed file at 1731576574370 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 30459879474abb0f5bba1f21ffa509ca in 41ms, sequenceid=32, compaction requested=false; wal=null at 1731576574376 (+6 ms)Cleaning up temporary data from old regions at 1731576574378 (+2 ms)Region opened successfully at 1731576574384 (+6 ms) 2024-11-14T09:29:34,412 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=417 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:59948 [Waiting for operation #54] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:40004 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:50096 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:57270 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:40458 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:56642 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=262 (was 262), ProcessCount=11 (was 11), AvailableMemoryMB=6760 (was 6752) - AvailableMemoryMB LEAK? - 2024-11-14T09:29:34,412 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-14T09:29:34,429 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=262, ProcessCount=11, AvailableMemoryMB=6758 2024-11-14T09:29:34,429 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-14T09:29:34,453 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:34,456 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:34,457 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:34,460 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39699251, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/hregion-39699251, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:34,480 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39699251/hregion-39699251.1731576574460, exclude list is [], retry=0 2024-11-14T09:29:34,484 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:34,484 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:34,484 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:34,487 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39699251/hregion-39699251.1731576574460 2024-11-14T09:29:34,488 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:34,488 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d9ece6ce7418486aa8b9c7c31c833a6a, NAME => 'testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/hbase 2024-11-14T09:29:34,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741907_1085 (size=68) 2024-11-14T09:29:34,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741907_1085 (size=68) 2024-11-14T09:29:34,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741907_1085 (size=68) 2024-11-14T09:29:34,501 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:34,503 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,504 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9ece6ce7418486aa8b9c7c31c833a6a columnFamilyName a 2024-11-14T09:29:34,504 DEBUG [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:34,505 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(327): Store=d9ece6ce7418486aa8b9c7c31c833a6a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:34,506 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,507 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9ece6ce7418486aa8b9c7c31c833a6a columnFamilyName b 2024-11-14T09:29:34,507 DEBUG [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:34,508 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(327): Store=d9ece6ce7418486aa8b9c7c31c833a6a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:34,508 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,509 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9ece6ce7418486aa8b9c7c31c833a6a columnFamilyName c 2024-11-14T09:29:34,509 DEBUG [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:34,510 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(327): Store=d9ece6ce7418486aa8b9c7c31c833a6a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:34,510 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,511 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,511 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,512 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,512 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,513 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:34,514 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,517 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:34,517 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9ece6ce7418486aa8b9c7c31c833a6a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66972364, jitterRate=-0.002034008502960205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:34,518 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9ece6ce7418486aa8b9c7c31c833a6a: Writing region info on filesystem at 1731576574502Initializing all the Stores at 1731576574502Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576574502Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576574503 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576574503Cleaning up temporary data from old regions at 1731576574512 (+9 ms)Region opened successfully at 1731576574518 (+6 ms) 2024-11-14T09:29:34,518 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d9ece6ce7418486aa8b9c7c31c833a6a, disabling compactions & flushes 2024-11-14T09:29:34,518 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. 2024-11-14T09:29:34,518 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. 2024-11-14T09:29:34,518 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. after waiting 0 ms 2024-11-14T09:29:34,518 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. 2024-11-14T09:29:34,519 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. 2024-11-14T09:29:34,519 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d9ece6ce7418486aa8b9c7c31c833a6a: Waiting for close lock at 1731576574518Disabling compacts and flushes for region at 1731576574518Disabling writes for close at 1731576574518Writing region close event to WAL at 1731576574519 (+1 ms)Closed at 1731576574519 2024-11-14T09:29:34,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741906_1084 (size=93) 2024-11-14T09:29:34,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741906_1084 (size=93) 2024-11-14T09:29:34,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741906_1084 (size=93) 2024-11-14T09:29:34,527 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:34,527 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39699251:(num 1731576574460) 2024-11-14T09:29:34,527 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:34,529 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:34,551 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530, exclude list is [], retry=0 2024-11-14T09:29:34,554 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:34,554 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:34,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:34,556 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530 2024-11-14T09:29:34,557 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:34,628 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d9ece6ce7418486aa8b9c7c31c833a6a, NAME => 'testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:34,631 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,631 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:34,631 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,631 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,636 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,637 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9ece6ce7418486aa8b9c7c31c833a6a columnFamilyName a 2024-11-14T09:29:34,637 DEBUG [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:34,638 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(327): Store=d9ece6ce7418486aa8b9c7c31c833a6a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:34,638 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,639 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9ece6ce7418486aa8b9c7c31c833a6a columnFamilyName b 2024-11-14T09:29:34,640 DEBUG [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:34,640 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(327): Store=d9ece6ce7418486aa8b9c7c31c833a6a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:34,640 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,641 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9ece6ce7418486aa8b9c7c31c833a6a columnFamilyName c 2024-11-14T09:29:34,641 DEBUG [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:34,642 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(327): Store=d9ece6ce7418486aa8b9c7c31c833a6a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:34,642 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,644 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,645 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,645 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,646 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:34,647 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,648 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9ece6ce7418486aa8b9c7c31c833a6a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74118868, jitterRate=0.10445719957351685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:34,648 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:34,648 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9ece6ce7418486aa8b9c7c31c833a6a: Running coprocessor pre-open hook at 1731576574631Writing region info on filesystem at 1731576574631Initializing all the Stores at 1731576574633 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576574633Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576574635 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576574635Cleaning up temporary data from old regions at 1731576574645 (+10 ms)Running coprocessor post-open hooks at 1731576574648 (+3 ms)Region opened successfully at 1731576574648 2024-11-14T09:29:34,660 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d9ece6ce7418486aa8b9c7c31c833a6a 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-14T09:29:34,661 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9ece6ce7418486aa8b9c7c31c833a6a/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:35,439 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:29:35,662 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9ece6ce7418486aa8b9c7c31c833a6a/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:36,662 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9ece6ce7418486aa8b9c7c31c833a6a/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:37,663 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9ece6ce7418486aa8b9c7c31c833a6a/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:38,663 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9ece6ce7418486aa8b9c7c31c833a6a/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:39,664 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9ece6ce7418486aa8b9c7c31c833a6a/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:40,665 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9ece6ce7418486aa8b9c7c31c833a6a/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:41,665 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9ece6ce7418486aa8b9c7c31c833a6a/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:42,202 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:29:42,666 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9ece6ce7418486aa8b9c7c31c833a6a/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:43,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-14T09:29:43,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-14T09:29:43,583 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-14T09:29:43,583 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-14T09:29:43,667 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9ece6ce7418486aa8b9c7c31c833a6a/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:43,668 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d9ece6ce7418486aa8b9c7c31c833a6a: 2024-11-14T09:29:43,668 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. 2024-11-14T09:29:43,680 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d9ece6ce7418486aa8b9c7c31c833a6a: 2024-11-14T09:29:43,680 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-14T09:29:43,680 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d9ece6ce7418486aa8b9c7c31c833a6a, disabling compactions & flushes 2024-11-14T09:29:43,680 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. 2024-11-14T09:29:43,680 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. 2024-11-14T09:29:43,680 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. after waiting 0 ms 2024-11-14T09:29:43,680 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. 2024-11-14T09:29:43,681 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. 2024-11-14T09:29:43,681 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. 2024-11-14T09:29:43,681 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d9ece6ce7418486aa8b9c7c31c833a6a: Waiting for close lock at 1731576583680Running coprocessor pre-close hooks at 1731576583680Disabling compacts and flushes for region at 1731576583680Disabling writes for close at 1731576583680Writing region close event to WAL at 1731576583681 (+1 ms)Running coprocessor post-close hooks at 1731576583681Closed at 1731576583681 2024-11-14T09:29:43,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741908_1086 (size=3941) 2024-11-14T09:29:43,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741908_1086 (size=3941) 2024-11-14T09:29:43,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741908_1086 (size=3941) 2024-11-14T09:29:43,700 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530, size=3.8 K (3941bytes) 2024-11-14T09:29:43,700 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530 2024-11-14T09:29:43,701 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530 after 0ms 2024-11-14T09:29:43,703 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530 took 3ms 2024-11-14T09:29:43,705 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530 so closing down 2024-11-14T09:29:43,705 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:43,706 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731576574530.temp 2024-11-14T09:29:43,707 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000004-wal.1731576574530.temp 2024-11-14T09:29:43,707 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:43,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741909_1087 (size=2983) 2024-11-14T09:29:43,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741909_1087 (size=2983) 2024-11-14T09:29:43,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741909_1087 (size=2983) 2024-11-14T09:29:43,714 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000004-wal.1731576574530.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:43,715 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000004-wal.1731576574530.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000026 2024-11-14T09:29:43,715 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-14T09:29:43,715 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530, journal: Splitting hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530, size=3.8 K (3941bytes) at 1731576583700Finishing writing output for hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530 so closing down at 1731576583705 (+5 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000004-wal.1731576574530.temp at 1731576583707 (+2 ms)3 split writer threads finished at 1731576583707Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000004-wal.1731576574530.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731576583714 (+7 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000004-wal.1731576574530.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000026 at 1731576583715 (+1 ms)Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1731576583715 2024-11-14T09:29:43,717 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576574530 to hdfs://localhost:34615/hbase/oldWALs/wal.1731576574530 2024-11-14T09:29:43,718 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000026 2024-11-14T09:29:43,718 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:43,719 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:43,732 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576583719, exclude list is [], retry=0 2024-11-14T09:29:43,735 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:43,735 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:43,735 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:43,736 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576583719 2024-11-14T09:29:43,737 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:43,737 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d9ece6ce7418486aa8b9c7c31c833a6a, NAME => 'testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:43,738 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,738 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:43,738 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,738 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,739 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,740 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9ece6ce7418486aa8b9c7c31c833a6a columnFamilyName a 2024-11-14T09:29:43,740 DEBUG [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:43,741 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(327): Store=d9ece6ce7418486aa8b9c7c31c833a6a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:43,741 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,741 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9ece6ce7418486aa8b9c7c31c833a6a columnFamilyName b 2024-11-14T09:29:43,742 DEBUG [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:43,742 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(327): Store=d9ece6ce7418486aa8b9c7c31c833a6a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:43,742 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,743 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9ece6ce7418486aa8b9c7c31c833a6a columnFamilyName c 2024-11-14T09:29:43,743 DEBUG [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:43,743 INFO [StoreOpener-d9ece6ce7418486aa8b9c7c31c833a6a-1 {}] regionserver.HStore(327): Store=d9ece6ce7418486aa8b9c7c31c833a6a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:43,743 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,744 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,745 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,746 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000026 2024-11-14T09:29:43,749 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000026 2024-11-14T09:29:43,749 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d9ece6ce7418486aa8b9c7c31c833a6a 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-14T09:29:43,763 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/.tmp/a/4f5bc320d4b74abe847b416d5bcdc0c2 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731576583672/Put/seqid=0 2024-11-14T09:29:43,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741911_1089 (size=5523) 2024-11-14T09:29:43,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741911_1089 (size=5523) 2024-11-14T09:29:43,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741911_1089 (size=5523) 2024-11-14T09:29:43,770 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/.tmp/a/4f5bc320d4b74abe847b416d5bcdc0c2 2024-11-14T09:29:43,789 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/.tmp/b/9df79c63fe8244309ec6fb28baae237d is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731576583668/Put/seqid=0 2024-11-14T09:29:43,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741912_1090 (size=5524) 2024-11-14T09:29:43,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741912_1090 (size=5524) 2024-11-14T09:29:43,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741912_1090 (size=5524) 2024-11-14T09:29:43,796 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/.tmp/b/9df79c63fe8244309ec6fb28baae237d 2024-11-14T09:29:43,816 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/.tmp/c/5d374e445eca4eec828d31ecda40b61b is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731576583670/Put/seqid=0 2024-11-14T09:29:43,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741913_1091 (size=5457) 2024-11-14T09:29:43,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741913_1091 (size=5457) 2024-11-14T09:29:43,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741913_1091 (size=5457) 2024-11-14T09:29:43,826 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/.tmp/c/5d374e445eca4eec828d31ecda40b61b 2024-11-14T09:29:43,833 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/.tmp/a/4f5bc320d4b74abe847b416d5bcdc0c2 as hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/a/4f5bc320d4b74abe847b416d5bcdc0c2 2024-11-14T09:29:43,837 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/a/4f5bc320d4b74abe847b416d5bcdc0c2, entries=7, sequenceid=26, filesize=5.4 K 2024-11-14T09:29:43,838 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/.tmp/b/9df79c63fe8244309ec6fb28baae237d as hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/b/9df79c63fe8244309ec6fb28baae237d 2024-11-14T09:29:43,842 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/b/9df79c63fe8244309ec6fb28baae237d, entries=7, sequenceid=26, filesize=5.4 K 2024-11-14T09:29:43,843 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/.tmp/c/5d374e445eca4eec828d31ecda40b61b as hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/c/5d374e445eca4eec828d31ecda40b61b 2024-11-14T09:29:43,848 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/c/5d374e445eca4eec828d31ecda40b61b, entries=6, sequenceid=26, filesize=5.3 K 2024-11-14T09:29:43,848 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for d9ece6ce7418486aa8b9c7c31c833a6a in 99ms, sequenceid=26, compaction requested=false; wal=null 2024-11-14T09:29:43,849 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/0000000000000000026 2024-11-14T09:29:43,850 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,850 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,851 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:43,852 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,854 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsAfterAbortingFlush/d9ece6ce7418486aa8b9c7c31c833a6a/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-14T09:29:43,855 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9ece6ce7418486aa8b9c7c31c833a6a; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65523743, jitterRate=-0.02362014353275299}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:43,855 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9ece6ce7418486aa8b9c7c31c833a6a 2024-11-14T09:29:43,855 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9ece6ce7418486aa8b9c7c31c833a6a: Running coprocessor pre-open hook at 1731576583738Writing region info on filesystem at 1731576583738Initializing all the Stores at 1731576583739 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576583739Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576583739Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576583739Obtaining lock to block concurrent updates at 1731576583749 (+10 ms)Preparing flush snapshotting stores in d9ece6ce7418486aa8b9c7c31c833a6a at 1731576583749Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731576583749Flushing stores of testReplayEditsAfterAbortingFlush,,1731576574454.d9ece6ce7418486aa8b9c7c31c833a6a. at 1731576583749Flushing d9ece6ce7418486aa8b9c7c31c833a6a/a: creating writer at 1731576583749Flushing d9ece6ce7418486aa8b9c7c31c833a6a/a: appending metadata at 1731576583763 (+14 ms)Flushing d9ece6ce7418486aa8b9c7c31c833a6a/a: closing flushed file at 1731576583763Flushing d9ece6ce7418486aa8b9c7c31c833a6a/b: creating writer at 1731576583775 (+12 ms)Flushing d9ece6ce7418486aa8b9c7c31c833a6a/b: appending metadata at 1731576583789 (+14 ms)Flushing d9ece6ce7418486aa8b9c7c31c833a6a/b: closing flushed file at 1731576583789Flushing d9ece6ce7418486aa8b9c7c31c833a6a/c: creating writer at 1731576583801 (+12 ms)Flushing d9ece6ce7418486aa8b9c7c31c833a6a/c: appending metadata at 1731576583816 (+15 ms)Flushing d9ece6ce7418486aa8b9c7c31c833a6a/c: closing flushed file at 1731576583816Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a2aac07: reopening flushed file at 1731576583832 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26a1128: reopening flushed file at 1731576583837 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@513bc3f0: reopening flushed file at 1731576583843 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for d9ece6ce7418486aa8b9c7c31c833a6a in 99ms, sequenceid=26, compaction requested=false; wal=null at 1731576583848 (+5 ms)Cleaning up temporary data from old regions at 1731576583850 (+2 ms)Running coprocessor post-open hooks at 1731576583855 (+5 ms)Region opened successfully at 1731576583855 2024-11-14T09:29:43,875 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 417), OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=229 (was 262), ProcessCount=11 (was 11), AvailableMemoryMB=6762 (was 6758) - AvailableMemoryMB LEAK? - 2024-11-14T09:29:43,875 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-14T09:29:43,888 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=6761 2024-11-14T09:29:43,888 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-14T09:29:43,904 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:43,906 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:43,907 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:43,910 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-14375410, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/hregion-14375410, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:43,931 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-14375410/hregion-14375410.1731576583910, exclude list is [], retry=0 2024-11-14T09:29:43,934 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:43,935 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:43,935 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:43,937 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-14375410/hregion-14375410.1731576583910 2024-11-14T09:29:43,940 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:43,940 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 827fb25216bad9b609c05d786fe05fb4, NAME => 'testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/hbase 2024-11-14T09:29:43,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741915_1093 (size=61) 2024-11-14T09:29:43,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741915_1093 (size=61) 2024-11-14T09:29:43,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741915_1093 (size=61) 2024-11-14T09:29:43,951 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:43,952 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,954 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 827fb25216bad9b609c05d786fe05fb4 columnFamilyName a 2024-11-14T09:29:43,954 DEBUG [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:43,954 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] regionserver.HStore(327): Store=827fb25216bad9b609c05d786fe05fb4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:43,954 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,955 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,955 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,956 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,956 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,957 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,959 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:43,960 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 827fb25216bad9b609c05d786fe05fb4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60786467, jitterRate=-0.09421105682849884}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T09:29:43,960 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 827fb25216bad9b609c05d786fe05fb4: Writing region info on filesystem at 1731576583951Initializing all the Stores at 1731576583952 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576583952Cleaning up temporary data from old regions at 1731576583956 (+4 ms)Region opened successfully at 1731576583960 (+4 ms) 2024-11-14T09:29:43,961 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 827fb25216bad9b609c05d786fe05fb4, disabling compactions & flushes 2024-11-14T09:29:43,961 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. 2024-11-14T09:29:43,961 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. 2024-11-14T09:29:43,961 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. after waiting 0 ms 2024-11-14T09:29:43,961 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. 2024-11-14T09:29:43,961 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. 2024-11-14T09:29:43,961 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 827fb25216bad9b609c05d786fe05fb4: Waiting for close lock at 1731576583961Disabling compacts and flushes for region at 1731576583961Disabling writes for close at 1731576583961Writing region close event to WAL at 1731576583961Closed at 1731576583961 2024-11-14T09:29:43,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741914_1092 (size=93) 2024-11-14T09:29:43,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741914_1092 (size=93) 2024-11-14T09:29:43,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741914_1092 (size=93) 2024-11-14T09:29:43,965 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:43,965 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-14375410:(num 1731576583910) 2024-11-14T09:29:43,965 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:43,967 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:43,981 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967, exclude list is [], retry=0 2024-11-14T09:29:43,984 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:43,984 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:43,984 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:43,986 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967 2024-11-14T09:29:43,986 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:43,986 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 827fb25216bad9b609c05d786fe05fb4, NAME => 'testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:43,986 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:43,987 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,987 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,988 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,989 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 827fb25216bad9b609c05d786fe05fb4 columnFamilyName a 2024-11-14T09:29:43,989 DEBUG [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:43,989 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] regionserver.HStore(327): Store=827fb25216bad9b609c05d786fe05fb4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:43,989 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,990 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,991 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,991 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,991 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,993 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:43,994 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 827fb25216bad9b609c05d786fe05fb4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75338477, jitterRate=0.12263078987598419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T09:29:43,995 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 827fb25216bad9b609c05d786fe05fb4: Writing region info on filesystem at 1731576583987Initializing all the Stores at 1731576583987Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576583987Cleaning up temporary data from old regions at 1731576583991 (+4 ms)Region opened successfully at 1731576583995 (+4 ms) 2024-11-14T09:29:44,003 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 827fb25216bad9b609c05d786fe05fb4, disabling compactions & flushes 2024-11-14T09:29:44,003 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. 2024-11-14T09:29:44,003 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. 2024-11-14T09:29:44,003 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. after waiting 0 ms 2024-11-14T09:29:44,003 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. 2024-11-14T09:29:44,003 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. 2024-11-14T09:29:44,003 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. 2024-11-14T09:29:44,003 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 827fb25216bad9b609c05d786fe05fb4: Waiting for close lock at 1731576584003Disabling compacts and flushes for region at 1731576584003Disabling writes for close at 1731576584003Writing region close event to WAL at 1731576584003Closed at 1731576584003 2024-11-14T09:29:44,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741916_1094 (size=1623) 2024-11-14T09:29:44,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741916_1094 (size=1623) 2024-11-14T09:29:44,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741916_1094 (size=1623) 2024-11-14T09:29:44,023 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967, size=1.6 K (1623bytes) 2024-11-14T09:29:44,023 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967 2024-11-14T09:29:44,024 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967 after 1ms 2024-11-14T09:29:44,026 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967 took 3ms 2024-11-14T09:29:44,028 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967 so closing down 2024-11-14T09:29:44,028 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:44,029 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731576583967.temp 2024-11-14T09:29:44,030 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000003-wal.1731576583967.temp 2024-11-14T09:29:44,030 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:44,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741917_1095 (size=1623) 2024-11-14T09:29:44,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741917_1095 (size=1623) 2024-11-14T09:29:44,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741917_1095 (size=1623) 2024-11-14T09:29:44,037 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000003-wal.1731576583967.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:44,039 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000003-wal.1731576583967.temp to hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000012 2024-11-14T09:29:44,039 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-14T09:29:44,039 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967, journal: Splitting hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967, size=1.6 K (1623bytes) at 1731576584023Finishing writing output for hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967 so closing down at 1731576584028 (+5 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000003-wal.1731576583967.temp at 1731576584030 (+2 ms)3 split writer threads finished at 1731576584030Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000003-wal.1731576583967.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731576584037 (+7 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000003-wal.1731576583967.temp to hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000012 at 1731576584039 (+2 ms)Processed 10 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1731576584039 2024-11-14T09:29:44,040 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576583967 to hdfs://localhost:34615/hbase/oldWALs/wal.1731576583967 2024-11-14T09:29:44,041 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000012 2024-11-14T09:29:44,408 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:44,411 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:44,425 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576584411, exclude list is [], retry=0 2024-11-14T09:29:44,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:44,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:44,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:44,430 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576584411 2024-11-14T09:29:44,430 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:44,430 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 827fb25216bad9b609c05d786fe05fb4, NAME => 'testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:44,430 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:44,430 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,431 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,433 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,435 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 827fb25216bad9b609c05d786fe05fb4 columnFamilyName a 2024-11-14T09:29:44,435 DEBUG [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:44,436 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] regionserver.HStore(327): Store=827fb25216bad9b609c05d786fe05fb4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:44,436 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,437 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,438 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,438 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000012 2024-11-14T09:29:44,441 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000012 2024-11-14T09:29:44,441 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 827fb25216bad9b609c05d786fe05fb4 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-14T09:29:44,460 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/.tmp/a/6ed13dfd93d4401f8cb922ab199a83ad is 79, key is testDatalossWhenInputError/a:x0/1731576583995/Put/seqid=0 2024-11-14T09:29:44,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741919_1097 (size=5808) 2024-11-14T09:29:44,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741919_1097 (size=5808) 2024-11-14T09:29:44,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741919_1097 (size=5808) 2024-11-14T09:29:44,466 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/.tmp/a/6ed13dfd93d4401f8cb922ab199a83ad 2024-11-14T09:29:44,475 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/.tmp/a/6ed13dfd93d4401f8cb922ab199a83ad as hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/a/6ed13dfd93d4401f8cb922ab199a83ad 2024-11-14T09:29:44,483 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/a/6ed13dfd93d4401f8cb922ab199a83ad, entries=10, sequenceid=12, filesize=5.7 K 2024-11-14T09:29:44,483 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 827fb25216bad9b609c05d786fe05fb4 in 42ms, sequenceid=12, compaction requested=false; wal=null 2024-11-14T09:29:44,484 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/0000000000000000012 2024-11-14T09:29:44,485 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,485 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,487 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,489 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-14T09:29:44,490 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 827fb25216bad9b609c05d786fe05fb4; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65653885, jitterRate=-0.02168087661266327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T09:29:44,491 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 827fb25216bad9b609c05d786fe05fb4: Writing region info on filesystem at 1731576584431Initializing all the Stores at 1731576584433 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576584433Obtaining lock to block concurrent updates at 1731576584441 (+8 ms)Preparing flush snapshotting stores in 827fb25216bad9b609c05d786fe05fb4 at 1731576584441Finished memstore snapshotting testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731576584441Flushing stores of testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4. at 1731576584441Flushing 827fb25216bad9b609c05d786fe05fb4/a: creating writer at 1731576584441Flushing 827fb25216bad9b609c05d786fe05fb4/a: appending metadata at 1731576584459 (+18 ms)Flushing 827fb25216bad9b609c05d786fe05fb4/a: closing flushed file at 1731576584459Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@188d2d4f: reopening flushed file at 1731576584473 (+14 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 827fb25216bad9b609c05d786fe05fb4 in 42ms, sequenceid=12, compaction requested=false; wal=null at 1731576584483 (+10 ms)Cleaning up temporary data from old regions at 1731576584485 (+2 ms)Region opened successfully at 1731576584491 (+6 ms) 2024-11-14T09:29:44,494 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 827fb25216bad9b609c05d786fe05fb4, NAME => 'testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:44,494 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731576583905.827fb25216bad9b609c05d786fe05fb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:44,495 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,495 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,496 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,497 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 827fb25216bad9b609c05d786fe05fb4 columnFamilyName a 2024-11-14T09:29:44,497 DEBUG [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:44,502 DEBUG [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/a/6ed13dfd93d4401f8cb922ab199a83ad 2024-11-14T09:29:44,502 INFO [StoreOpener-827fb25216bad9b609c05d786fe05fb4-1 {}] regionserver.HStore(327): Store=827fb25216bad9b609c05d786fe05fb4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:44,502 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,503 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,504 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,504 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,504 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,505 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 827fb25216bad9b609c05d786fe05fb4 2024-11-14T09:29:44,507 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testDatalossWhenInputError/827fb25216bad9b609c05d786fe05fb4/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-14T09:29:44,508 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 827fb25216bad9b609c05d786fe05fb4; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61810833, jitterRate=-0.07894681394100189}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T09:29:44,508 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 827fb25216bad9b609c05d786fe05fb4: Writing region info on filesystem at 1731576584495Initializing all the Stores at 1731576584496 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576584496Cleaning up temporary data from old regions at 1731576584504 (+8 ms)Region opened successfully at 1731576584508 (+4 ms) 2024-11-14T09:29:44,525 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:43988 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:44408 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:34344 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:44474 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:44046 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-376496606-172.17.0.2-1731576549671:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:34282 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=229 (was 229), ProcessCount=11 (was 11), AvailableMemoryMB=6669 (was 6761) 2024-11-14T09:29:44,525 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-14T09:29:44,537 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=6667 2024-11-14T09:29:44,537 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-14T09:29:44,554 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:44,556 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:44,556 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:44,558 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-41477396, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/hregion-41477396, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:44,573 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-41477396/hregion-41477396.1731576584559, exclude list is [], retry=0 2024-11-14T09:29:44,575 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:44,575 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:44,576 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:44,580 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-41477396/hregion-41477396.1731576584559 2024-11-14T09:29:44,580 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:44,580 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1126e9e4ab1a99dc052556b51371e1f6, NAME => 'testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/hbase 2024-11-14T09:29:44,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741921_1099 (size=63) 2024-11-14T09:29:44,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741921_1099 (size=63) 2024-11-14T09:29:44,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741921_1099 (size=63) 2024-11-14T09:29:44,591 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:44,593 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,594 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126e9e4ab1a99dc052556b51371e1f6 columnFamilyName a 2024-11-14T09:29:44,594 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:44,594 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(327): Store=1126e9e4ab1a99dc052556b51371e1f6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:44,595 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,596 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126e9e4ab1a99dc052556b51371e1f6 columnFamilyName b 2024-11-14T09:29:44,596 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:44,598 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(327): Store=1126e9e4ab1a99dc052556b51371e1f6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:44,598 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,600 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126e9e4ab1a99dc052556b51371e1f6 columnFamilyName c 2024-11-14T09:29:44,600 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:44,601 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(327): Store=1126e9e4ab1a99dc052556b51371e1f6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:44,601 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,602 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,602 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,604 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,604 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,605 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:44,606 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,609 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:44,609 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1126e9e4ab1a99dc052556b51371e1f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69576788, jitterRate=0.03677493333816528}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:44,611 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1126e9e4ab1a99dc052556b51371e1f6: Writing region info on filesystem at 1731576584591Initializing all the Stores at 1731576584592 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576584592Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576584592Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576584592Cleaning up temporary data from old regions at 1731576584604 (+12 ms)Region opened successfully at 1731576584610 (+6 ms) 2024-11-14T09:29:44,611 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1126e9e4ab1a99dc052556b51371e1f6, disabling compactions & flushes 2024-11-14T09:29:44,611 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. 2024-11-14T09:29:44,611 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. 2024-11-14T09:29:44,611 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. after waiting 0 ms 2024-11-14T09:29:44,611 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. 2024-11-14T09:29:44,612 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. 2024-11-14T09:29:44,613 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1126e9e4ab1a99dc052556b51371e1f6: Waiting for close lock at 1731576584611Disabling compacts and flushes for region at 1731576584611Disabling writes for close at 1731576584611Writing region close event to WAL at 1731576584612 (+1 ms)Closed at 1731576584612 2024-11-14T09:29:44,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741920_1098 (size=93) 2024-11-14T09:29:44,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741920_1098 (size=93) 2024-11-14T09:29:44,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741920_1098 (size=93) 2024-11-14T09:29:44,617 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:44,617 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-41477396:(num 1731576584559) 2024-11-14T09:29:44,617 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:44,619 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:44,633 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619, exclude list is [], retry=0 2024-11-14T09:29:44,636 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:44,636 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:44,636 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:44,637 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 2024-11-14T09:29:44,638 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:44,638 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1126e9e4ab1a99dc052556b51371e1f6, NAME => 'testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:44,638 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:44,638 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,638 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,639 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,640 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126e9e4ab1a99dc052556b51371e1f6 columnFamilyName a 2024-11-14T09:29:44,640 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:44,641 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(327): Store=1126e9e4ab1a99dc052556b51371e1f6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:44,641 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,642 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126e9e4ab1a99dc052556b51371e1f6 columnFamilyName b 2024-11-14T09:29:44,642 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:44,643 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(327): Store=1126e9e4ab1a99dc052556b51371e1f6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:44,643 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,644 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126e9e4ab1a99dc052556b51371e1f6 columnFamilyName c 2024-11-14T09:29:44,644 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:44,644 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(327): Store=1126e9e4ab1a99dc052556b51371e1f6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:44,644 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,645 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,646 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,647 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,647 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,647 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:44,649 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:44,649 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1126e9e4ab1a99dc052556b51371e1f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74788283, jitterRate=0.11443226039409637}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:44,650 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1126e9e4ab1a99dc052556b51371e1f6: Writing region info on filesystem at 1731576584638Initializing all the Stores at 1731576584639 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576584639Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576584639Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576584639Cleaning up temporary data from old regions at 1731576584647 (+8 ms)Region opened successfully at 1731576584650 (+3 ms) 2024-11-14T09:29:44,654 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731576584654/Put/seqid=0 2024-11-14T09:29:44,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741923_1101 (size=4875) 2024-11-14T09:29:44,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741923_1101 (size=4875) 2024-11-14T09:29:44,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741923_1101 (size=4875) 2024-11-14T09:29:44,664 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731576584663/Put/seqid=0 2024-11-14T09:29:44,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741924_1102 (size=4875) 2024-11-14T09:29:44,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741924_1102 (size=4875) 2024-11-14T09:29:44,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741924_1102 (size=4875) 2024-11-14T09:29:44,677 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731576584677/Put/seqid=0 2024-11-14T09:29:44,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741925_1103 (size=4875) 2024-11-14T09:29:44,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741925_1103 (size=4875) 2024-11-14T09:29:44,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741925_1103 (size=4875) 2024-11-14T09:29:44,687 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 1126e9e4ab1a99dc052556b51371e1f6/a 2024-11-14T09:29:44,690 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-14T09:29:44,690 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T09:29:44,690 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 1126e9e4ab1a99dc052556b51371e1f6/a 2024-11-14T09:29:44,693 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-14T09:29:44,693 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T09:29:44,694 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 1126e9e4ab1a99dc052556b51371e1f6/a 2024-11-14T09:29:44,696 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-14T09:29:44,696 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T09:29:44,697 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1126e9e4ab1a99dc052556b51371e1f6 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-14T09:29:44,714 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/.tmp/a/f529405e1fd94e858333f3a3095a5918 is 55, key is testCompactedBulkLoadedFiles/a:a/1731576584651/Put/seqid=0 2024-11-14T09:29:44,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741926_1104 (size=5107) 2024-11-14T09:29:44,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741926_1104 (size=5107) 2024-11-14T09:29:44,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741926_1104 (size=5107) 2024-11-14T09:29:44,724 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/.tmp/a/f529405e1fd94e858333f3a3095a5918 2024-11-14T09:29:44,730 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/.tmp/a/f529405e1fd94e858333f3a3095a5918 as hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/f529405e1fd94e858333f3a3095a5918 2024-11-14T09:29:44,735 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/f529405e1fd94e858333f3a3095a5918, entries=1, sequenceid=4, filesize=5.0 K 2024-11-14T09:29:44,736 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 1126e9e4ab1a99dc052556b51371e1f6 in 39ms, sequenceid=4, compaction requested=false 2024-11-14T09:29:44,736 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1126e9e4ab1a99dc052556b51371e1f6: 2024-11-14T09:29:44,737 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_ 2024-11-14T09:29:44,738 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_ 2024-11-14T09:29:44,739 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/4a30c472b5914cf09a5f8cd26457186c_SeqId_4_ 2024-11-14T09:29:44,740 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile0 into 1126e9e4ab1a99dc052556b51371e1f6/a as hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_ - updating store file list. 2024-11-14T09:29:44,745 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T09:29:44,745 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_ into 1126e9e4ab1a99dc052556b51371e1f6/a 2024-11-14T09:29:44,745 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile0 into 1126e9e4ab1a99dc052556b51371e1f6/a (new location: hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_) 2024-11-14T09:29:44,746 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile1 into 1126e9e4ab1a99dc052556b51371e1f6/a as hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_ - updating store file list. 2024-11-14T09:29:44,751 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T09:29:44,751 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_ into 1126e9e4ab1a99dc052556b51371e1f6/a 2024-11-14T09:29:44,751 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile1 into 1126e9e4ab1a99dc052556b51371e1f6/a (new location: hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_) 2024-11-14T09:29:44,752 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile2 into 1126e9e4ab1a99dc052556b51371e1f6/a as hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/4a30c472b5914cf09a5f8cd26457186c_SeqId_4_ - updating store file list. 2024-11-14T09:29:44,756 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4a30c472b5914cf09a5f8cd26457186c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T09:29:44,756 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/4a30c472b5914cf09a5f8cd26457186c_SeqId_4_ into 1126e9e4ab1a99dc052556b51371e1f6/a 2024-11-14T09:29:44,756 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34615/hbase/testCompactedBulkLoadedFiles/hfile2 into 1126e9e4ab1a99dc052556b51371e1f6/a (new location: hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/4a30c472b5914cf09a5f8cd26457186c_SeqId_4_) 2024-11-14T09:29:44,762 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:29:44,763 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1126e9e4ab1a99dc052556b51371e1f6/a is initiating major compaction (all files) 2024-11-14T09:29:44,763 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1126e9e4ab1a99dc052556b51371e1f6/a in testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. 2024-11-14T09:29:44,763 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/f529405e1fd94e858333f3a3095a5918, hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_, hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_, hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/4a30c472b5914cf09a5f8cd26457186c_SeqId_4_] into tmpdir=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/.tmp, totalSize=19.3 K 2024-11-14T09:29:44,763 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f529405e1fd94e858333f3a3095a5918, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731576584651 2024-11-14T09:29:44,764 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T09:29:44,764 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T09:29:44,764 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4a30c472b5914cf09a5f8cd26457186c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T09:29:44,801 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/.tmp/a/c9066f48350c43d2902ebc866489315c is 55, key is testCompactedBulkLoadedFiles/a:a/1731576584651/Put/seqid=0 2024-11-14T09:29:44,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741927_1105 (size=6154) 2024-11-14T09:29:44,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741927_1105 (size=6154) 2024-11-14T09:29:44,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741927_1105 (size=6154) 2024-11-14T09:29:44,817 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/.tmp/a/c9066f48350c43d2902ebc866489315c as hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/c9066f48350c43d2902ebc866489315c 2024-11-14T09:29:44,822 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 1126e9e4ab1a99dc052556b51371e1f6/a of 1126e9e4ab1a99dc052556b51371e1f6 into c9066f48350c43d2902ebc866489315c(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:29:44,822 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1126e9e4ab1a99dc052556b51371e1f6: 2024-11-14T09:29:44,822 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-14T09:29:44,822 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-14T09:29:44,855 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619, size=0 (0bytes) 2024-11-14T09:29:44,856 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 might be still open, length is 0 2024-11-14T09:29:44,856 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 2024-11-14T09:29:44,856 WARN [IPC Server handler 0 on default port 34615 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-14T09:29:44,856 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 after 0ms 2024-11-14T09:29:47,806 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:44500 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41175:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44500 dst: /127.0.0.1:41175 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41175 remote=/127.0.0.1:44500]. Total timeout mills is 60000, 57017 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:47,807 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:34382 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37113:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34382 dst: /127.0.0.1:37113 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:47,807 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:44102 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43439:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44102 dst: /127.0.0.1:43439 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741922_1106 (size=1659) 2024-11-14T09:29:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741922_1106 (size=1659) 2024-11-14T09:29:47,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741922_1106 (size=1659) 2024-11-14T09:29:48,857 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 after 4001ms 2024-11-14T09:29:48,860 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 took 4005ms 2024-11-14T09:29:48,862 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619; continuing. 2024-11-14T09:29:48,862 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 so closing down 2024-11-14T09:29:48,862 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:48,863 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731576584619.temp 2024-11-14T09:29:48,865 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000003-wal.1731576584619.temp 2024-11-14T09:29:48,865 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:48,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741928_1107 (size=698) 2024-11-14T09:29:48,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741928_1107 (size=698) 2024-11-14T09:29:48,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741928_1107 (size=698) 2024-11-14T09:29:48,877 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000003-wal.1731576584619.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:48,879 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000003-wal.1731576584619.temp to hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000008 2024-11-14T09:29:48,879 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T09:29:48,879 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619, journal: Splitting hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619, size=0 (0bytes) at 1731576584856Finishing writing output for hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 so closing down at 1731576588862 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000003-wal.1731576584619.temp at 1731576588865 (+3 ms)3 split writer threads finished at 1731576588865Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000003-wal.1731576584619.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731576588877 (+12 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000003-wal.1731576584619.temp to hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000008 at 1731576588879 (+2 ms)Processed 5 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619, size=0, length=0, corrupted=false, cancelled=false at 1731576588879 2024-11-14T09:29:48,881 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 to hdfs://localhost:34615/hbase/oldWALs/wal.1731576584619 2024-11-14T09:29:48,881 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000008 2024-11-14T09:29:48,882 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:48,883 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:48,896 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576588883, exclude list is [], retry=0 2024-11-14T09:29:48,899 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:48,899 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:48,899 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:48,901 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576588883 2024-11-14T09:29:48,901 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:48,901 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1126e9e4ab1a99dc052556b51371e1f6, NAME => 'testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:48,902 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:48,902 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,902 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,903 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,904 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126e9e4ab1a99dc052556b51371e1f6 columnFamilyName a 2024-11-14T09:29:48,904 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:48,910 DEBUG [StoreFileOpener-1126e9e4ab1a99dc052556b51371e1f6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T09:29:48,910 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_ 2024-11-14T09:29:48,914 DEBUG [StoreFileOpener-1126e9e4ab1a99dc052556b51371e1f6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4a30c472b5914cf09a5f8cd26457186c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T09:29:48,914 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/4a30c472b5914cf09a5f8cd26457186c_SeqId_4_ 2024-11-14T09:29:48,918 DEBUG [StoreFileOpener-1126e9e4ab1a99dc052556b51371e1f6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T09:29:48,918 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_ 2024-11-14T09:29:48,921 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/c9066f48350c43d2902ebc866489315c 2024-11-14T09:29:48,925 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/f529405e1fd94e858333f3a3095a5918 2024-11-14T09:29:48,926 WARN [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@501e14b3 2024-11-14T09:29:48,926 WARN [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/4a30c472b5914cf09a5f8cd26457186c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@501e14b3 2024-11-14T09:29:48,926 WARN [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@501e14b3 2024-11-14T09:29:48,926 WARN [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/f529405e1fd94e858333f3a3095a5918 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@501e14b3 2024-11-14T09:29:48,926 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_, hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/4a30c472b5914cf09a5f8cd26457186c_SeqId_4_, hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_, hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/f529405e1fd94e858333f3a3095a5918] to archive 2024-11-14T09:29:48,927 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:29:48,929 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_ to hdfs://localhost:34615/hbase/archive/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_ 2024-11-14T09:29:48,930 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/4a30c472b5914cf09a5f8cd26457186c_SeqId_4_ to hdfs://localhost:34615/hbase/archive/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/4a30c472b5914cf09a5f8cd26457186c_SeqId_4_ 2024-11-14T09:29:48,931 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_ to hdfs://localhost:34615/hbase/archive/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_ 2024-11-14T09:29:48,932 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/f529405e1fd94e858333f3a3095a5918 to hdfs://localhost:34615/hbase/archive/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/a/f529405e1fd94e858333f3a3095a5918 2024-11-14T09:29:48,933 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(327): Store=1126e9e4ab1a99dc052556b51371e1f6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:48,933 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,934 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126e9e4ab1a99dc052556b51371e1f6 columnFamilyName b 2024-11-14T09:29:48,934 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:48,934 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(327): Store=1126e9e4ab1a99dc052556b51371e1f6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:48,934 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,935 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1126e9e4ab1a99dc052556b51371e1f6 columnFamilyName c 2024-11-14T09:29:48,935 DEBUG [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:48,936 INFO [StoreOpener-1126e9e4ab1a99dc052556b51371e1f6-1 {}] regionserver.HStore(327): Store=1126e9e4ab1a99dc052556b51371e1f6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:48,936 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,936 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,938 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,938 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000008 2024-11-14T09:29:48,942 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 1126e9e4ab1a99dc052556b51371e1f6 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "1126e9e4ab1a99dc052556b51371e1f6" family_name: "a" compaction_input: "f529405e1fd94e858333f3a3095a5918" compaction_input: "041fb14d46a643fb9df54a44f3bb9b93_SeqId_4_" compaction_input: "5a68bb5d7a5443f0997a2f1c5db96ee0_SeqId_4_" compaction_input: "4a30c472b5914cf09a5f8cd26457186c_SeqId_4_" compaction_output: "c9066f48350c43d2902ebc866489315c" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-14T09:29:48,942 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-14T09:29:48,943 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000008 2024-11-14T09:29:48,943 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/0000000000000000008 2024-11-14T09:29:48,945 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,945 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,946 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:48,947 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1126e9e4ab1a99dc052556b51371e1f6 2024-11-14T09:29:48,949 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testCompactedBulkLoadedFiles/1126e9e4ab1a99dc052556b51371e1f6/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-14T09:29:48,950 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1126e9e4ab1a99dc052556b51371e1f6; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62310140, jitterRate=-0.0715065598487854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:48,950 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1126e9e4ab1a99dc052556b51371e1f6: Writing region info on filesystem at 1731576588902Initializing all the Stores at 1731576588902Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576588902Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576588903 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576588903Cleaning up temporary data from old regions at 1731576588945 (+42 ms)Region opened successfully at 1731576588950 (+5 ms) 2024-11-14T09:29:48,954 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1126e9e4ab1a99dc052556b51371e1f6, disabling compactions & flushes 2024-11-14T09:29:48,954 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. 2024-11-14T09:29:48,954 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. 2024-11-14T09:29:48,954 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. after waiting 0 ms 2024-11-14T09:29:48,954 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. 2024-11-14T09:29:48,955 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731576584554.1126e9e4ab1a99dc052556b51371e1f6. 2024-11-14T09:29:48,955 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1126e9e4ab1a99dc052556b51371e1f6: Waiting for close lock at 1731576588954Disabling compacts and flushes for region at 1731576588954Disabling writes for close at 1731576588954Writing region close event to WAL at 1731576588955 (+1 ms)Closed at 1731576588955 2024-11-14T09:29:48,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741929_1108 (size=93) 2024-11-14T09:29:48,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741929_1108 (size=93) 2024-11-14T09:29:48,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741929_1108 (size=93) 2024-11-14T09:29:48,962 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:48,962 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731576588883) 2024-11-14T09:29:48,979 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=437 (was 425) Potentially hanging thread: IPC Client (1017487743) connection to localhost/127.0.0.1:34615 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:34615 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1157591708_22 at /127.0.0.1:34454 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1017487743) connection to localhost/127.0.0.1:45747 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1157591708_22 at /127.0.0.1:44144 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1157591708_22 at /127.0.0.1:44562 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45747 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=218 (was 229), ProcessCount=11 (was 11), AvailableMemoryMB=6598 (was 6667) 2024-11-14T09:29:48,980 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-14T09:29:48,997 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=437, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=218, ProcessCount=11, AvailableMemoryMB=6597 2024-11-14T09:29:48,997 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-14T09:29:49,021 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:49,023 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:29:49,024 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:29:49,027 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-51057248, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/hregion-51057248, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:49,047 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-51057248/hregion-51057248.1731576589027, exclude list is [], retry=0 2024-11-14T09:29:49,051 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:49,051 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:49,052 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:49,056 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-51057248/hregion-51057248.1731576589027 2024-11-14T09:29:49,059 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845)] 2024-11-14T09:29:49,059 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2381a60bb9e56680ec0b29beba24d714, NAME => 'testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34615/hbase 2024-11-14T09:29:49,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741931_1110 (size=67) 2024-11-14T09:29:49,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741931_1110 (size=67) 2024-11-14T09:29:49,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741931_1110 (size=67) 2024-11-14T09:29:49,080 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:49,082 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,084 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName a 2024-11-14T09:29:49,084 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:49,084 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:49,084 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,086 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName b 2024-11-14T09:29:49,086 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:49,086 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:49,086 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,087 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName c 2024-11-14T09:29:49,087 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:49,088 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:49,088 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,089 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,089 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,090 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,090 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,090 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:49,092 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,094 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:29:49,094 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2381a60bb9e56680ec0b29beba24d714; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59220378, jitterRate=-0.11754760146141052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:49,094 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2381a60bb9e56680ec0b29beba24d714: Writing region info on filesystem at 1731576589081Initializing all the Stores at 1731576589081Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576589082 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576589082Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576589082Cleaning up temporary data from old regions at 1731576589090 (+8 ms)Region opened successfully at 1731576589094 (+4 ms) 2024-11-14T09:29:49,095 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2381a60bb9e56680ec0b29beba24d714, disabling compactions & flushes 2024-11-14T09:29:49,095 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:49,095 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:49,095 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. after waiting 0 ms 2024-11-14T09:29:49,095 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:49,095 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:49,095 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2381a60bb9e56680ec0b29beba24d714: Waiting for close lock at 1731576589095Disabling compacts and flushes for region at 1731576589095Disabling writes for close at 1731576589095Writing region close event to WAL at 1731576589095Closed at 1731576589095 2024-11-14T09:29:49,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741930_1109 (size=93) 2024-11-14T09:29:49,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741930_1109 (size=93) 2024-11-14T09:29:49,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741930_1109 (size=93) 2024-11-14T09:29:49,101 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:49,101 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-51057248:(num 1731576589027) 2024-11-14T09:29:49,101 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:49,103 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:49,123 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103, exclude list is [], retry=0 2024-11-14T09:29:49,126 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:49,127 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:49,127 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:49,129 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103 2024-11-14T09:29:49,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:49,129 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2381a60bb9e56680ec0b29beba24d714, NAME => 'testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:49,129 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:49,129 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,129 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,131 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,131 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName a 2024-11-14T09:29:49,131 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:49,132 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:49,132 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,133 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName b 2024-11-14T09:29:49,133 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:49,133 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:49,133 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,134 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName c 2024-11-14T09:29:49,134 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:49,134 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:49,134 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,135 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,136 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,137 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,137 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,137 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:49,138 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,139 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2381a60bb9e56680ec0b29beba24d714; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62346810, jitterRate=-0.070960134267807}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:49,139 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2381a60bb9e56680ec0b29beba24d714: Writing region info on filesystem at 1731576589129Initializing all the Stores at 1731576589130 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576589130Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576589130Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576589130Cleaning up temporary data from old regions at 1731576589137 (+7 ms)Region opened successfully at 1731576589139 (+2 ms) 2024-11-14T09:29:49,146 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2381a60bb9e56680ec0b29beba24d714 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-14T09:29:49,162 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/a/9b1e9ef96b784e1f81cc707148278f7c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731576589139/Put/seqid=0 2024-11-14T09:29:49,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741933_1112 (size=5958) 2024-11-14T09:29:49,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741933_1112 (size=5958) 2024-11-14T09:29:49,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741933_1112 (size=5958) 2024-11-14T09:29:49,170 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/a/9b1e9ef96b784e1f81cc707148278f7c 2024-11-14T09:29:49,176 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/a/9b1e9ef96b784e1f81cc707148278f7c as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/a/9b1e9ef96b784e1f81cc707148278f7c 2024-11-14T09:29:49,182 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/a/9b1e9ef96b784e1f81cc707148278f7c, entries=10, sequenceid=13, filesize=5.8 K 2024-11-14T09:29:49,183 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2381a60bb9e56680ec0b29beba24d714 in 37ms, sequenceid=13, compaction requested=false 2024-11-14T09:29:49,183 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2381a60bb9e56680ec0b29beba24d714: 2024-11-14T09:29:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2381a60bb9e56680ec0b29beba24d714, disabling compactions & flushes 2024-11-14T09:29:49,200 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. after waiting 0 ms 2024-11-14T09:29:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:49,201 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:29:49,201 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:49,201 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:49,201 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2381a60bb9e56680ec0b29beba24d714: Waiting for close lock at 1731576589200Disabling compacts and flushes for region at 1731576589200Disabling writes for close at 1731576589200Writing region close event to WAL at 1731576589201 (+1 ms)Closed at 1731576589201 2024-11-14T09:29:49,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741932_1111 (size=5869) 2024-11-14T09:29:49,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741932_1111 (size=5869) 2024-11-14T09:29:49,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741932_1111 (size=5869) 2024-11-14T09:29:49,220 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103, size=5.7 K (5869bytes) 2024-11-14T09:29:49,221 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103 2024-11-14T09:29:49,221 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103 after 0ms 2024-11-14T09:29:49,223 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103 took 3ms 2024-11-14T09:29:49,225 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103 so closing down 2024-11-14T09:29:49,225 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:49,226 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731576589103.temp 2024-11-14T09:29:49,227 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000003-wal.1731576589103.temp 2024-11-14T09:29:49,227 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:49,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741934_1113 (size=5223) 2024-11-14T09:29:49,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741934_1113 (size=5223) 2024-11-14T09:29:49,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741934_1113 (size=5223) 2024-11-14T09:29:49,235 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000003-wal.1731576589103.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:49,236 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000003-wal.1731576589103.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000035 2024-11-14T09:29:49,236 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-14T09:29:49,236 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103, journal: Splitting hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103, size=5.7 K (5869bytes) at 1731576589221Finishing writing output for hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103 so closing down at 1731576589225 (+4 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000003-wal.1731576589103.temp at 1731576589227 (+2 ms)3 split writer threads finished at 1731576589228 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000003-wal.1731576589103.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731576589235 (+7 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000003-wal.1731576589103.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000035 at 1731576589236 (+1 ms)Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1731576589236 2024-11-14T09:29:49,238 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589103 to hdfs://localhost:34615/hbase/oldWALs/wal.1731576589103 2024-11-14T09:29:49,238 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000035 2024-11-14T09:29:49,239 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:49,240 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:49,257 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240, exclude list is [], retry=0 2024-11-14T09:29:49,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:49,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:49,260 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:49,261 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 2024-11-14T09:29:49,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35591:35591),(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477)] 2024-11-14T09:29:49,262 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2381a60bb9e56680ec0b29beba24d714, NAME => 'testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:29:49,262 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:49,262 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,262 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,263 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,264 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName a 2024-11-14T09:29:49,264 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:49,269 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/a/9b1e9ef96b784e1f81cc707148278f7c 2024-11-14T09:29:49,269 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:49,270 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,270 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName b 2024-11-14T09:29:49,270 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:49,271 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:49,271 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,272 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName c 2024-11-14T09:29:49,272 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:49,272 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:49,272 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,273 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,274 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,274 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000035 2024-11-14T09:29:49,277 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000035 2024-11-14T09:29:49,277 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2381a60bb9e56680ec0b29beba24d714 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-14T09:29:49,292 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/b/0921920399da4e55887e3c18e0793cab is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731576589183/Put/seqid=0 2024-11-14T09:29:49,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741936_1115 (size=5958) 2024-11-14T09:29:49,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741936_1115 (size=5958) 2024-11-14T09:29:49,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741936_1115 (size=5958) 2024-11-14T09:29:49,301 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/b/0921920399da4e55887e3c18e0793cab 2024-11-14T09:29:49,326 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/c/98619495f735420c8f46777472205ab3 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731576589190/Put/seqid=0 2024-11-14T09:29:49,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741937_1116 (size=5958) 2024-11-14T09:29:49,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741937_1116 (size=5958) 2024-11-14T09:29:49,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741937_1116 (size=5958) 2024-11-14T09:29:49,333 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/c/98619495f735420c8f46777472205ab3 2024-11-14T09:29:49,339 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/b/0921920399da4e55887e3c18e0793cab as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/b/0921920399da4e55887e3c18e0793cab 2024-11-14T09:29:49,343 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/b/0921920399da4e55887e3c18e0793cab, entries=10, sequenceid=35, filesize=5.8 K 2024-11-14T09:29:49,344 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/c/98619495f735420c8f46777472205ab3 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/c/98619495f735420c8f46777472205ab3 2024-11-14T09:29:49,348 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/c/98619495f735420c8f46777472205ab3, entries=10, sequenceid=35, filesize=5.8 K 2024-11-14T09:29:49,349 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 2381a60bb9e56680ec0b29beba24d714 in 72ms, sequenceid=35, compaction requested=false; wal=null 2024-11-14T09:29:49,349 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000035 2024-11-14T09:29:49,350 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,350 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,351 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:49,352 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:49,354 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-14T09:29:49,355 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2381a60bb9e56680ec0b29beba24d714; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64574479, jitterRate=-0.03776527941226959}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:49,355 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2381a60bb9e56680ec0b29beba24d714: Writing region info on filesystem at 1731576589262Initializing all the Stores at 1731576589263 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576589263Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576589263Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576589263Obtaining lock to block concurrent updates at 1731576589277 (+14 ms)Preparing flush snapshotting stores in 2381a60bb9e56680ec0b29beba24d714 at 1731576589277Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731576589278 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. at 1731576589278Flushing 2381a60bb9e56680ec0b29beba24d714/b: creating writer at 1731576589278Flushing 2381a60bb9e56680ec0b29beba24d714/b: appending metadata at 1731576589291 (+13 ms)Flushing 2381a60bb9e56680ec0b29beba24d714/b: closing flushed file at 1731576589291Flushing 2381a60bb9e56680ec0b29beba24d714/c: creating writer at 1731576589305 (+14 ms)Flushing 2381a60bb9e56680ec0b29beba24d714/c: appending metadata at 1731576589325 (+20 ms)Flushing 2381a60bb9e56680ec0b29beba24d714/c: closing flushed file at 1731576589325Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cd1edac: reopening flushed file at 1731576589338 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e73bd6c: reopening flushed file at 1731576589343 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 2381a60bb9e56680ec0b29beba24d714 in 72ms, sequenceid=35, compaction requested=false; wal=null at 1731576589349 (+6 ms)Cleaning up temporary data from old regions at 1731576589350 (+1 ms)Region opened successfully at 1731576589355 (+5 ms) 2024-11-14T09:29:49,422 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240, size=0 (0bytes) 2024-11-14T09:29:49,422 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 might be still open, length is 0 2024-11-14T09:29:49,422 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 2024-11-14T09:29:49,423 WARN [IPC Server handler 4 on default port 34615 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-14T09:29:49,423 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 after 1ms 2024-11-14T09:29:50,990 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:44234 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43439:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44234 dst: /127.0.0.1:43439 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43439 remote=/127.0.0.1:44234]. Total timeout mills is 60000, 58393 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:50,991 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:34530 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37113:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34530 dst: /127.0.0.1:37113 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:50,991 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1354416917_22 at /127.0.0.1:44620 [Receiving block BP-376496606-172.17.0.2-1731576549671:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41175:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44620 dst: /127.0.0.1:41175 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:50,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741935_1117 (size=5215) 2024-11-14T09:29:50,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741935_1117 (size=5215) 2024-11-14T09:29:50,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741935_1117 (size=5215) 2024-11-14T09:29:53,424 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 after 4002ms 2024-11-14T09:29:53,427 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 took 4005ms 2024-11-14T09:29:53,428 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240; continuing. 2024-11-14T09:29:53,428 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 so closing down 2024-11-14T09:29:53,429 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T09:29:53,430 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731576589240.temp 2024-11-14T09:29:53,431 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000037-wal.1731576589240.temp 2024-11-14T09:29:53,431 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T09:29:53,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741938_1118 (size=5223) 2024-11-14T09:29:53,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741938_1118 (size=5223) 2024-11-14T09:29:53,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741938_1118 (size=5223) 2024-11-14T09:29:53,437 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000037-wal.1731576589240.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T09:29:53,439 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000037-wal.1731576589240.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000066 2024-11-14T09:29:53,439 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T09:29:53,439 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240, journal: Splitting hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240, size=0 (0bytes) at 1731576589422Finishing writing output for hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 so closing down at 1731576593429 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000037-wal.1731576589240.temp at 1731576593431 (+2 ms)3 split writer threads finished at 1731576593431Closed recovered edits writer path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000037-wal.1731576589240.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731576593437 (+6 ms)Rename recovered edits hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000037-wal.1731576589240.temp to hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000066 at 1731576593439 (+2 ms)Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240, size=0, length=0, corrupted=false, cancelled=false at 1731576593439 2024-11-14T09:29:53,441 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 to hdfs://localhost:34615/hbase/oldWALs/wal.1731576589240 2024-11-14T09:29:53,441 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000066 2024-11-14T09:29:53,442 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T09:29:53,443 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34615/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020, archiveDir=hdfs://localhost:34615/hbase/oldWALs, maxLogs=32 2024-11-14T09:29:53,457 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576593443, exclude list is [], retry=0 2024-11-14T09:29:53,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37113,DS-5e08a043-ecd3-4be0-878e-2a81fea495ce,DISK] 2024-11-14T09:29:53,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41175,DS-860268cd-ff3b-48ac-aa99-1273241384d4,DISK] 2024-11-14T09:29:53,460 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43439,DS-a51058c8-5b64-4eb0-9983-3592a1dfd434,DISK] 2024-11-14T09:29:53,461 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576593443 2024-11-14T09:29:53,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39845:39845),(127.0.0.1/127.0.0.1:40477:40477),(127.0.0.1/127.0.0.1:35591:35591)] 2024-11-14T09:29:53,462 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:29:53,463 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:53,463 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName a 2024-11-14T09:29:53,464 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:53,468 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/a/9b1e9ef96b784e1f81cc707148278f7c 2024-11-14T09:29:53,468 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:53,468 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:53,469 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName b 2024-11-14T09:29:53,469 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:53,473 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/b/0921920399da4e55887e3c18e0793cab 2024-11-14T09:29:53,473 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:53,473 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:53,474 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2381a60bb9e56680ec0b29beba24d714 columnFamilyName c 2024-11-14T09:29:53,474 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:29:53,477 DEBUG [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/c/98619495f735420c8f46777472205ab3 2024-11-14T09:29:53,477 INFO [StoreOpener-2381a60bb9e56680ec0b29beba24d714-1 {}] regionserver.HStore(327): Store=2381a60bb9e56680ec0b29beba24d714/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:29:53,478 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:53,478 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:53,479 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:53,480 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000066 2024-11-14T09:29:53,486 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000066 2024-11-14T09:29:53,486 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2381a60bb9e56680ec0b29beba24d714 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-14T09:29:53,501 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/a/63ee24b357304cfeaf79e41f51f80cdb is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731576589363/Put/seqid=0 2024-11-14T09:29:53,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741940_1120 (size=5958) 2024-11-14T09:29:53,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741940_1120 (size=5958) 2024-11-14T09:29:53,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741940_1120 (size=5958) 2024-11-14T09:29:53,508 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/a/63ee24b357304cfeaf79e41f51f80cdb 2024-11-14T09:29:53,526 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/b/cd7a6bebbb1644f58ad6d931536f7873 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731576589369/Put/seqid=0 2024-11-14T09:29:53,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741941_1121 (size=5958) 2024-11-14T09:29:53,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741941_1121 (size=5958) 2024-11-14T09:29:53,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741941_1121 (size=5958) 2024-11-14T09:29:53,532 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/b/cd7a6bebbb1644f58ad6d931536f7873 2024-11-14T09:29:53,550 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/c/c408948ae50f49b49805a5906ca070b7 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731576589376/Put/seqid=0 2024-11-14T09:29:53,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741942_1122 (size=5958) 2024-11-14T09:29:53,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741942_1122 (size=5958) 2024-11-14T09:29:53,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741942_1122 (size=5958) 2024-11-14T09:29:53,557 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/c/c408948ae50f49b49805a5906ca070b7 2024-11-14T09:29:53,562 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/a/63ee24b357304cfeaf79e41f51f80cdb as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/a/63ee24b357304cfeaf79e41f51f80cdb 2024-11-14T09:29:53,565 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/a/63ee24b357304cfeaf79e41f51f80cdb, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T09:29:53,566 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/b/cd7a6bebbb1644f58ad6d931536f7873 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/b/cd7a6bebbb1644f58ad6d931536f7873 2024-11-14T09:29:53,570 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/b/cd7a6bebbb1644f58ad6d931536f7873, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T09:29:53,571 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/.tmp/c/c408948ae50f49b49805a5906ca070b7 as hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/c/c408948ae50f49b49805a5906ca070b7 2024-11-14T09:29:53,575 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/c/c408948ae50f49b49805a5906ca070b7, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T09:29:53,575 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2381a60bb9e56680ec0b29beba24d714 in 89ms, sequenceid=66, compaction requested=false; wal=null 2024-11-14T09:29:53,576 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/0000000000000000066 2024-11-14T09:29:53,577 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:53,577 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:53,577 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T09:29:53,578 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2381a60bb9e56680ec0b29beba24d714 2024-11-14T09:29:53,580 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/hbase/data/default/testReplayEditsWrittenViaHRegion/2381a60bb9e56680ec0b29beba24d714/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-14T09:29:53,581 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2381a60bb9e56680ec0b29beba24d714; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69231151, jitterRate=0.03162454068660736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T09:29:53,581 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2381a60bb9e56680ec0b29beba24d714: Writing region info on filesystem at 1731576593462Initializing all the Stores at 1731576593462Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576593462Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576593463 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731576593463Obtaining lock to block concurrent updates at 1731576593486 (+23 ms)Preparing flush snapshotting stores in 2381a60bb9e56680ec0b29beba24d714 at 1731576593486Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731576593486Flushing stores of testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. at 1731576593486Flushing 2381a60bb9e56680ec0b29beba24d714/a: creating writer at 1731576593486Flushing 2381a60bb9e56680ec0b29beba24d714/a: appending metadata at 1731576593500 (+14 ms)Flushing 2381a60bb9e56680ec0b29beba24d714/a: closing flushed file at 1731576593500Flushing 2381a60bb9e56680ec0b29beba24d714/b: creating writer at 1731576593511 (+11 ms)Flushing 2381a60bb9e56680ec0b29beba24d714/b: appending metadata at 1731576593525 (+14 ms)Flushing 2381a60bb9e56680ec0b29beba24d714/b: closing flushed file at 1731576593525Flushing 2381a60bb9e56680ec0b29beba24d714/c: creating writer at 1731576593536 (+11 ms)Flushing 2381a60bb9e56680ec0b29beba24d714/c: appending metadata at 1731576593550 (+14 ms)Flushing 2381a60bb9e56680ec0b29beba24d714/c: closing flushed file at 1731576593550Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d44378a: reopening flushed file at 1731576593561 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e813f86: reopening flushed file at 1731576593566 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@251ab314: reopening flushed file at 1731576593570 (+4 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2381a60bb9e56680ec0b29beba24d714 in 89ms, sequenceid=66, compaction requested=false; wal=null at 1731576593575 (+5 ms)Cleaning up temporary data from old regions at 1731576593577 (+2 ms)Region opened successfully at 1731576593581 (+4 ms) 2024-11-14T09:29:53,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-14T09:29:53,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-14T09:29:53,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-14T09:29:53,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-14T09:29:53,593 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2381a60bb9e56680ec0b29beba24d714, disabling compactions & flushes 2024-11-14T09:29:53,593 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:53,593 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:53,593 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. after waiting 0 ms 2024-11-14T09:29:53,593 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:53,594 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731576589022.2381a60bb9e56680ec0b29beba24d714. 2024-11-14T09:29:53,594 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2381a60bb9e56680ec0b29beba24d714: Waiting for close lock at 1731576593593Disabling compacts and flushes for region at 1731576593593Disabling writes for close at 1731576593593Writing region close event to WAL at 1731576593594 (+1 ms)Closed at 1731576593594 2024-11-14T09:29:53,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741939_1119 (size=93) 2024-11-14T09:29:53,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741939_1119 (size=93) 2024-11-14T09:29:53,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741939_1119 (size=93) 2024-11-14T09:29:53,598 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T09:29:53,598 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731576593443) 2024-11-14T09:29:53,613 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=446 (was 437) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1824249594_22 at /127.0.0.1:45180 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:34615 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1017487743) connection to localhost/127.0.0.1:34615 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1824249594_22 at /127.0.0.1:58104 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1824249594_22 at /127.0.0.1:48902 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 218), ProcessCount=11 (was 11), AvailableMemoryMB=6717 (was 6597) - AvailableMemoryMB LEAK? - 2024-11-14T09:29:53,613 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-14T09:29:53,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T09:29:53,613 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:29:53,613 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:29:53,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:29:53,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:29:53,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:29:53,614 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T09:29:53,614 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=633453724, stopped=false 2024-11-14T09:29:53,615 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=83f56b55f2af,43303,1731576553096 2024-11-14T09:29:53,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T09:29:53,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T09:29:53,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T09:29:53,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:53,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:53,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:53,617 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T09:29:53,617 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:29:53,617 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:29:53,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:29:53,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:29:53,618 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:29:53,618 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '83f56b55f2af,32943,1731576553877' ***** 2024-11-14T09:29:53,618 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:29:53,618 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T09:29:53,618 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '83f56b55f2af,43391,1731576553984' ***** 2024-11-14T09:29:53,618 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T09:29:53,618 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T09:29:53,618 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T09:29:53,619 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T09:29:53,619 INFO [RS:1;83f56b55f2af:43391 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T09:29:53,619 INFO [RS:1;83f56b55f2af:43391 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T09:29:53,619 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T09:29:53,619 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(959): stopping server 83f56b55f2af,43391,1731576553984 2024-11-14T09:29:53,619 INFO [RS:0;83f56b55f2af:32943 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T09:29:53,619 INFO [RS:0;83f56b55f2af:32943 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T09:29:53,619 INFO [RS:1;83f56b55f2af:43391 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:29:53,619 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(3091): Received CLOSE for e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:53,619 INFO [RS:1;83f56b55f2af:43391 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;83f56b55f2af:43391. 2024-11-14T09:29:53,619 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(959): stopping server 83f56b55f2af,32943,1731576553877 2024-11-14T09:29:53,619 DEBUG [RS:1;83f56b55f2af:43391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:29:53,619 INFO [RS:0;83f56b55f2af:32943 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:29:53,619 DEBUG [RS:1;83f56b55f2af:43391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:29:53,619 INFO [RS:0;83f56b55f2af:32943 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;83f56b55f2af:32943. 2024-11-14T09:29:53,619 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T09:29:53,619 DEBUG [RS:0;83f56b55f2af:32943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:29:53,619 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T09:29:53,619 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T09:29:53,619 DEBUG [RS:0;83f56b55f2af:32943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:29:53,619 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e5dc7f00d707752df5f2aa388572824b, disabling compactions & flushes 2024-11-14T09:29:53,619 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T09:29:53,619 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:53,619 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T09:29:53,619 DEBUG [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(1325): Online Regions={e5dc7f00d707752df5f2aa388572824b=testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b.} 2024-11-14T09:29:53,619 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:53,619 DEBUG [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(1351): Waiting on e5dc7f00d707752df5f2aa388572824b 2024-11-14T09:29:53,620 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. after waiting 0 ms 2024-11-14T09:29:53,620 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:53,620 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T09:29:53,620 DEBUG [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T09:29:53,620 DEBUG [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:29:53,620 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:29:53,620 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:29:53,620 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:29:53,620 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:29:53,620 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:29:53,620 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-14T09:29:53,624 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e5dc7f00d707752df5f2aa388572824b/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-14T09:29:53,625 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/info/bace44ee55764c5fa2fabab525d3a8ed is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b./info:regioninfo/1731576573373/Put/seqid=0 2024-11-14T09:29:53,625 INFO [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:53,625 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e5dc7f00d707752df5f2aa388572824b: Waiting for close lock at 1731576593619Running coprocessor pre-close hooks at 1731576593619Disabling compacts and flushes for region at 1731576593619Disabling writes for close at 1731576593620 (+1 ms)Writing region close event to WAL at 1731576593621 (+1 ms)Running coprocessor post-close hooks at 1731576593625 (+4 ms)Closed at 1731576593625 2024-11-14T09:29:53,625 DEBUG [RS_CLOSE_REGION-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731576569522.e5dc7f00d707752df5f2aa388572824b. 2024-11-14T09:29:53,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741943_1123 (size=6778) 2024-11-14T09:29:53,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741943_1123 (size=6778) 2024-11-14T09:29:53,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741943_1123 (size=6778) 2024-11-14T09:29:53,633 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/info/bace44ee55764c5fa2fabab525d3a8ed 2024-11-14T09:29:53,639 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/.tmp/info/bace44ee55764c5fa2fabab525d3a8ed as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/info/bace44ee55764c5fa2fabab525d3a8ed 2024-11-14T09:29:53,644 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/info/bace44ee55764c5fa2fabab525d3a8ed, entries=8, sequenceid=23, filesize=6.6 K 2024-11-14T09:29:53,645 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=23, compaction requested=false 2024-11-14T09:29:53,649 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-14T09:29:53,649 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T09:29:53,650 INFO [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:29:53,650 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731576593620Running coprocessor pre-close hooks at 1731576593620Disabling compacts and flushes for region at 1731576593620Disabling writes for close at 1731576593620Obtaining lock to block concurrent updates at 1731576593620Preparing flush snapshotting stores in 1588230740 at 1731576593620Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1731576593620Flushing stores of hbase:meta,,1.1588230740 at 1731576593621 (+1 ms)Flushing 1588230740/info: creating writer at 1731576593621Flushing 1588230740/info: appending metadata at 1731576593625 (+4 ms)Flushing 1588230740/info: closing flushed file at 1731576593625Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dc41d72: reopening flushed file at 1731576593638 (+13 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=23, compaction requested=false at 1731576593645 (+7 ms)Writing region close event to WAL at 1731576593646 (+1 ms)Running coprocessor post-close hooks at 1731576593649 (+3 ms)Closed at 1731576593650 (+1 ms) 2024-11-14T09:29:53,650 DEBUG [RS_CLOSE_META-regionserver/83f56b55f2af:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T09:29:53,703 INFO [regionserver/83f56b55f2af:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:29:53,703 INFO [regionserver/83f56b55f2af:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:29:53,820 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(976): stopping server 83f56b55f2af,32943,1731576553877; all regions closed. 2024-11-14T09:29:53,820 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(976): stopping server 83f56b55f2af,43391,1731576553984; all regions closed. 2024-11-14T09:29:53,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741893_1071 (size=2422) 2024-11-14T09:29:53,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741893_1071 (size=2422) 2024-11-14T09:29:53,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741833_1009 (size=3433) 2024-11-14T09:29:53,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741833_1009 (size=3433) 2024-11-14T09:29:53,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741893_1071 (size=2422) 2024-11-14T09:29:53,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741833_1009 (size=3433) 2024-11-14T09:29:53,827 DEBUG [RS:0;83f56b55f2af:32943 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/oldWALs 2024-11-14T09:29:53,827 INFO [RS:0;83f56b55f2af:32943 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 83f56b55f2af%2C32943%2C1731576553877:(num 1731576555518) 2024-11-14T09:29:53,827 DEBUG [RS:0;83f56b55f2af:32943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:29:53,827 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:29:53,827 INFO [RS:0;83f56b55f2af:32943 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:29:53,827 INFO [RS:0;83f56b55f2af:32943 {}] hbase.ChoreService(370): Chore service for: regionserver/83f56b55f2af:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T09:29:53,827 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T09:29:53,827 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T09:29:53,827 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T09:29:53,827 INFO [regionserver/83f56b55f2af:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:29:53,827 INFO [RS:0;83f56b55f2af:32943 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:29:53,827 DEBUG [RS:1;83f56b55f2af:43391 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/oldWALs 2024-11-14T09:29:53,827 INFO [RS:1;83f56b55f2af:43391 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 83f56b55f2af%2C43391%2C1731576553984.meta:.meta(num 1731576572547) 2024-11-14T09:29:53,827 INFO [RS:0;83f56b55f2af:32943 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32943 2024-11-14T09:29:53,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:29:53,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741834_1010 (size=93) 2024-11-14T09:29:53,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741834_1010 (size=93) 2024-11-14T09:29:53,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/83f56b55f2af,32943,1731576553877 2024-11-14T09:29:53,830 INFO [RS:0;83f56b55f2af:32943 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:29:53,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741834_1010 (size=93) 2024-11-14T09:29:53,831 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [83f56b55f2af,32943,1731576553877] 2024-11-14T09:29:53,833 DEBUG [RS:1;83f56b55f2af:43391 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/oldWALs 2024-11-14T09:29:53,833 INFO [RS:1;83f56b55f2af:43391 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 83f56b55f2af%2C43391%2C1731576553984:(num 1731576555518) 2024-11-14T09:29:53,833 DEBUG [RS:1;83f56b55f2af:43391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:29:53,833 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:29:53,833 INFO [RS:1;83f56b55f2af:43391 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:29:53,833 INFO [RS:1;83f56b55f2af:43391 {}] hbase.ChoreService(370): Chore service for: regionserver/83f56b55f2af:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T09:29:53,833 INFO [RS:1;83f56b55f2af:43391 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:29:53,833 INFO [regionserver/83f56b55f2af:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:29:53,833 INFO [RS:1;83f56b55f2af:43391 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43391 2024-11-14T09:29:53,833 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/83f56b55f2af,32943,1731576553877 already deleted, retry=false 2024-11-14T09:29:53,834 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 83f56b55f2af,32943,1731576553877 expired; onlineServers=1 2024-11-14T09:29:53,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/83f56b55f2af,43391,1731576553984 2024-11-14T09:29:53,835 INFO [RS:1;83f56b55f2af:43391 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:29:53,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:29:53,836 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [83f56b55f2af,43391,1731576553984] 2024-11-14T09:29:53,838 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/83f56b55f2af,43391,1731576553984 already deleted, retry=false 2024-11-14T09:29:53,838 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 83f56b55f2af,43391,1731576553984 expired; onlineServers=0 2024-11-14T09:29:53,838 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '83f56b55f2af,43303,1731576553096' ***** 2024-11-14T09:29:53,838 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T09:29:53,838 INFO [M:0;83f56b55f2af:43303 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:29:53,838 INFO [M:0;83f56b55f2af:43303 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:29:53,838 DEBUG [M:0;83f56b55f2af:43303 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T09:29:53,838 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T09:29:53,838 DEBUG [M:0;83f56b55f2af:43303 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T09:29:53,838 DEBUG [master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.small.0-1731576555077 {}] cleaner.HFileCleaner(306): Exit Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.small.0-1731576555077,5,FailOnTimeoutGroup] 2024-11-14T09:29:53,838 DEBUG [master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.large.0-1731576555074 {}] cleaner.HFileCleaner(306): Exit Thread[master/83f56b55f2af:0:becomeActiveMaster-HFileCleaner.large.0-1731576555074,5,FailOnTimeoutGroup] 2024-11-14T09:29:53,838 INFO [M:0;83f56b55f2af:43303 {}] hbase.ChoreService(370): Chore service for: master/83f56b55f2af:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T09:29:53,839 INFO [M:0;83f56b55f2af:43303 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:29:53,839 DEBUG [M:0;83f56b55f2af:43303 {}] master.HMaster(1795): Stopping service threads 2024-11-14T09:29:53,839 INFO [M:0;83f56b55f2af:43303 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T09:29:53,839 INFO [M:0;83f56b55f2af:43303 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T09:29:53,839 INFO [M:0;83f56b55f2af:43303 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T09:29:53,839 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T09:29:53,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T09:29:53,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:29:53,841 DEBUG [M:0;83f56b55f2af:43303 {}] zookeeper.ZKUtil(347): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T09:29:53,841 WARN [M:0;83f56b55f2af:43303 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T09:29:53,841 INFO [M:0;83f56b55f2af:43303 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/.lastflushedseqids 2024-11-14T09:29:53,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741944_1124 (size=119) 2024-11-14T09:29:53,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741944_1124 (size=119) 2024-11-14T09:29:53,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741944_1124 (size=119) 2024-11-14T09:29:53,854 INFO [M:0;83f56b55f2af:43303 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T09:29:53,855 INFO [M:0;83f56b55f2af:43303 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T09:29:53,855 DEBUG [M:0;83f56b55f2af:43303 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T09:29:53,855 INFO [M:0;83f56b55f2af:43303 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:29:53,855 DEBUG [M:0;83f56b55f2af:43303 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:29:53,855 DEBUG [M:0;83f56b55f2af:43303 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T09:29:53,855 DEBUG [M:0;83f56b55f2af:43303 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:29:53,855 INFO [M:0;83f56b55f2af:43303 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-14T09:29:53,871 DEBUG [M:0;83f56b55f2af:43303 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/845088712ea44e6fa224db31e4ebaab3 is 82, key is hbase:meta,,1/info:regioninfo/1731576572714/Put/seqid=0 2024-11-14T09:29:53,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741945_1125 (size=6063) 2024-11-14T09:29:53,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741945_1125 (size=6063) 2024-11-14T09:29:53,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741945_1125 (size=6063) 2024-11-14T09:29:53,877 INFO [M:0;83f56b55f2af:43303 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/845088712ea44e6fa224db31e4ebaab3 2024-11-14T09:29:53,896 DEBUG [M:0;83f56b55f2af:43303 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c80253ac6e0f43e68a2cd6aa8f2962f2 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731576569946/Put/seqid=0 2024-11-14T09:29:53,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741946_1126 (size=7907) 2024-11-14T09:29:53,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741946_1126 (size=7907) 2024-11-14T09:29:53,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741946_1126 (size=7907) 2024-11-14T09:29:53,902 INFO [M:0;83f56b55f2af:43303 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c80253ac6e0f43e68a2cd6aa8f2962f2 2024-11-14T09:29:53,906 INFO [M:0;83f56b55f2af:43303 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c80253ac6e0f43e68a2cd6aa8f2962f2 2024-11-14T09:29:53,920 DEBUG [M:0;83f56b55f2af:43303 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c2c7292f274e4711833c4bed46b8b2e6 is 69, key is 83f56b55f2af,32943,1731576553877/rs:state/1731576555264/Put/seqid=0 2024-11-14T09:29:53,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741947_1127 (size=5445) 2024-11-14T09:29:53,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741947_1127 (size=5445) 2024-11-14T09:29:53,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741947_1127 (size=5445) 2024-11-14T09:29:53,927 INFO [M:0;83f56b55f2af:43303 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c2c7292f274e4711833c4bed46b8b2e6 2024-11-14T09:29:53,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:29:53,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32943-0x10115cf45f40001, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:29:53,932 INFO [RS:0;83f56b55f2af:32943 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:29:53,932 INFO [M:0;83f56b55f2af:43303 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c2c7292f274e4711833c4bed46b8b2e6 2024-11-14T09:29:53,932 INFO [RS:0;83f56b55f2af:32943 {}] regionserver.HRegionServer(1031): Exiting; stopping=83f56b55f2af,32943,1731576553877; zookeeper connection closed. 2024-11-14T09:29:53,932 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4165d305 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4165d305 2024-11-14T09:29:53,933 DEBUG [M:0;83f56b55f2af:43303 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/845088712ea44e6fa224db31e4ebaab3 as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/845088712ea44e6fa224db31e4ebaab3 2024-11-14T09:29:53,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:29:53,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43391-0x10115cf45f40002, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:29:53,937 INFO [RS:1;83f56b55f2af:43391 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:29:53,937 INFO [RS:1;83f56b55f2af:43391 {}] regionserver.HRegionServer(1031): Exiting; stopping=83f56b55f2af,43391,1731576553984; zookeeper connection closed. 2024-11-14T09:29:53,937 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6db698a1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6db698a1 2024-11-14T09:29:53,937 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-14T09:29:53,939 INFO [M:0;83f56b55f2af:43303 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/845088712ea44e6fa224db31e4ebaab3, entries=14, sequenceid=207, filesize=5.9 K 2024-11-14T09:29:53,939 DEBUG [M:0;83f56b55f2af:43303 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c80253ac6e0f43e68a2cd6aa8f2962f2 as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c80253ac6e0f43e68a2cd6aa8f2962f2 2024-11-14T09:29:53,944 INFO [M:0;83f56b55f2af:43303 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c80253ac6e0f43e68a2cd6aa8f2962f2 2024-11-14T09:29:53,945 INFO [M:0;83f56b55f2af:43303 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c80253ac6e0f43e68a2cd6aa8f2962f2, entries=21, sequenceid=207, filesize=7.7 K 2024-11-14T09:29:53,946 DEBUG [M:0;83f56b55f2af:43303 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c2c7292f274e4711833c4bed46b8b2e6 as hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c2c7292f274e4711833c4bed46b8b2e6 2024-11-14T09:29:53,950 INFO [M:0;83f56b55f2af:43303 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c2c7292f274e4711833c4bed46b8b2e6 2024-11-14T09:29:53,951 INFO [M:0;83f56b55f2af:43303 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34615/user/jenkins/test-data/33bcc275-af02-8da0-2090-a86dc3e1e963/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c2c7292f274e4711833c4bed46b8b2e6, entries=3, sequenceid=207, filesize=5.3 K 2024-11-14T09:29:53,952 INFO [M:0;83f56b55f2af:43303 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=207, compaction requested=false 2024-11-14T09:29:53,955 INFO [M:0;83f56b55f2af:43303 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:29:53,955 DEBUG [M:0;83f56b55f2af:43303 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731576593855Disabling compacts and flushes for region at 1731576593855Disabling writes for close at 1731576593855Obtaining lock to block concurrent updates at 1731576593855Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731576593855Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1731576593855Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731576593856 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731576593856Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731576593870 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731576593870Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731576593881 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731576593895 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731576593895Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731576593906 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731576593920 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731576593920Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35d088b1: reopening flushed file at 1731576593932 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ee44b27: reopening flushed file at 1731576593939 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a0da60f: reopening flushed file at 1731576593945 (+6 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=207, compaction requested=false at 1731576593952 (+7 ms)Writing region close event to WAL at 1731576593955 (+3 ms)Closed at 1731576593955 2024-11-14T09:29:53,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43439 is added to blk_1073741830_1006 (size=99436) 2024-11-14T09:29:53,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37113 is added to blk_1073741830_1006 (size=99436) 2024-11-14T09:29:53,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41175 is added to blk_1073741830_1006 (size=99436) 2024-11-14T09:29:53,958 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:29:53,958 INFO [M:0;83f56b55f2af:43303 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T09:29:53,959 INFO [M:0;83f56b55f2af:43303 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43303 2024-11-14T09:29:53,959 INFO [M:0;83f56b55f2af:43303 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:29:54,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:29:54,061 INFO [M:0;83f56b55f2af:43303 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:29:54,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43303-0x10115cf45f40000, quorum=127.0.0.1:59004, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:29:54,068 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576583719 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576583719 (inode 16688) Holder DFSClient_NONMAPREDUCE_1354416917_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731576574452/wal.1731576583719 (inode 16688) Holder DFSClient_NONMAPREDUCE_1354416917_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T09:29:54,071 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576574291 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576574291 (inode 16665) Holder DFSClient_NONMAPREDUCE_1354416917_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731576573596/wal.1731576574291 (inode 16665) Holder DFSClient_NONMAPREDUCE_1354416917_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T09:29:54,071 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731576564601/wal.1731576564804 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:54,073 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576584411 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576584411 (inode 16714) Holder DFSClient_NONMAPREDUCE_1354416917_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731576583903/wal.1731576584411 (inode 16714) Holder DFSClient_NONMAPREDUCE_1354416917_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T09:29:54,074 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731576584553/wal.1731576584619 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:54,074 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731576589020/wal.1731576589240 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:54,074 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731576557899/wal.1731576557989 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:29:54,076 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal.1731576557739 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal.1731576557739 (inode 16485) Holder DFSClient_NONMAPREDUCE_1354416917_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731576557427/wal.1731576557739 (inode 16485) Holder DFSClient_NONMAPREDUCE_1354416917_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T09:29:54,078 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal.1731576556735 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal.1731576556735 (inode 16462) Holder DFSClient_NONMAPREDUCE_1354416917_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731576556369/wal.1731576556735 (inode 16462) Holder DFSClient_NONMAPREDUCE_1354416917_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T09:29:54,082 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@37ee96b7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:29:54,084 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3b82d1a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T09:29:54,084 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T09:29:54,085 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@73d49bd7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T09:29:54,085 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1576ddbd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/hadoop.log.dir/,STOPPED} 2024-11-14T09:29:54,087 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T09:29:54,087 WARN [BP-376496606-172.17.0.2-1731576549671 heartbeating to localhost/127.0.0.1:34615 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T09:29:54,087 WARN [BP-376496606-172.17.0.2-1731576549671 heartbeating to localhost/127.0.0.1:34615 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-376496606-172.17.0.2-1731576549671 (Datanode Uuid 4f0a233f-d3b7-47c2-ae66-d7e05aff4523) service to localhost/127.0.0.1:34615 2024-11-14T09:29:54,087 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T09:29:54,089 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data5/current/BP-376496606-172.17.0.2-1731576549671 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:29:54,089 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data6/current/BP-376496606-172.17.0.2-1731576549671 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:29:54,089 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T09:29:54,091 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@23205bfa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:29:54,091 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1df40b66{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T09:29:54,091 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T09:29:54,091 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@363c876e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T09:29:54,091 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5deb6e4f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/hadoop.log.dir/,STOPPED} 2024-11-14T09:29:54,092 WARN [BP-376496606-172.17.0.2-1731576549671 heartbeating to localhost/127.0.0.1:34615 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T09:29:54,093 WARN [BP-376496606-172.17.0.2-1731576549671 heartbeating to localhost/127.0.0.1:34615 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-376496606-172.17.0.2-1731576549671 (Datanode Uuid a74e73bc-bb0c-4473-aeb7-28477c3b69ea) service to localhost/127.0.0.1:34615 2024-11-14T09:29:54,093 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T09:29:54,093 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data3/current/BP-376496606-172.17.0.2-1731576549671 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:29:54,093 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data4/current/BP-376496606-172.17.0.2-1731576549671 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:29:54,093 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T09:29:54,093 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T09:29:54,095 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c8aad7c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:29:54,096 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d06fb64{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T09:29:54,096 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T09:29:54,096 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8f47f6d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T09:29:54,096 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@66bf9098{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/hadoop.log.dir/,STOPPED} 2024-11-14T09:29:54,097 WARN [BP-376496606-172.17.0.2-1731576549671 heartbeating to localhost/127.0.0.1:34615 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T09:29:54,097 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T09:29:54,097 WARN [BP-376496606-172.17.0.2-1731576549671 heartbeating to localhost/127.0.0.1:34615 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-376496606-172.17.0.2-1731576549671 (Datanode Uuid cd6cfbf2-5c89-44ac-8692-c27ea5b719ab) service to localhost/127.0.0.1:34615 2024-11-14T09:29:54,097 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T09:29:54,098 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data1/current/BP-376496606-172.17.0.2-1731576549671 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:29:54,098 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/cluster_f912bdbb-6d2c-a7f1-fe55-7bb587e31152/data/data2/current/BP-376496606-172.17.0.2-1731576549671 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:29:54,098 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T09:29:54,104 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@25dc94e0{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T09:29:54,105 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7cc108fc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T09:29:54,105 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T09:29:54,105 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@377439aa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T09:29:54,105 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39b35319{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3ec2dc58-974c-c5b7-0cf2-10b64bedcf7d/hadoop.log.dir/,STOPPED} 2024-11-14T09:29:54,115 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T09:29:54,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down