2024-11-26 07:27:05,511 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-26 07:27:05,522 main DEBUG Took 0.008959 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 07:27:05,522 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 07:27:05,523 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 07:27:05,523 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 07:27:05,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,531 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 07:27:05,542 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,544 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,545 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,545 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,545 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,546 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,546 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,547 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,547 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,547 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,548 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,548 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,549 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,549 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,549 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,550 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,550 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,550 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,551 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,551 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,551 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,552 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:27:05,553 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,553 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 07:27:05,554 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:27:05,556 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 07:27:05,557 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 07:27:05,557 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 07:27:05,558 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 07:27:05,559 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 07:27:05,567 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 07:27:05,569 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 07:27:05,571 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 07:27:05,571 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 07:27:05,572 main DEBUG createAppenders(={Console}) 2024-11-26 07:27:05,572 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-26 07:27:05,573 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-26 07:27:05,573 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-26 07:27:05,573 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 07:27:05,574 main DEBUG OutputStream closed 2024-11-26 07:27:05,574 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 07:27:05,574 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 07:27:05,574 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-26 07:27:05,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 07:27:05,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 07:27:05,642 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 07:27:05,643 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 07:27:05,644 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 07:27:05,644 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 07:27:05,645 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 07:27:05,645 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 07:27:05,645 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 07:27:05,646 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 07:27:05,646 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 07:27:05,646 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 07:27:05,647 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 07:27:05,647 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 07:27:05,647 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 07:27:05,648 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 07:27:05,648 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 07:27:05,649 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 07:27:05,651 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 07:27:05,651 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-26 07:27:05,651 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 07:27:05,652 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-26T07:27:05,909 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78 2024-11-26 07:27:05,911 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 07:27:05,911 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T07:27:05,921 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-26T07:27:05,926 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-11-26T07:27:05,946 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T07:27:06,002 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T07:27:06,002 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T07:27:06,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T07:27:06,028 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01, deleteOnExit=true 2024-11-26T07:27:06,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T07:27:06,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/test.cache.data in system properties and HBase conf 2024-11-26T07:27:06,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T07:27:06,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/hadoop.log.dir in system properties and HBase conf 2024-11-26T07:27:06,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T07:27:06,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T07:27:06,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T07:27:06,107 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T07:27:06,199 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:27:06,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T07:27:06,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T07:27:06,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T07:27:06,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T07:27:06,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T07:27:06,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T07:27:06,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T07:27:06,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T07:27:06,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T07:27:06,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/nfs.dump.dir in system properties and HBase conf 2024-11-26T07:27:06,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/java.io.tmpdir in system properties and HBase conf 2024-11-26T07:27:06,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T07:27:06,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T07:27:06,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T07:27:07,009 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T07:27:07,088 INFO [Time-limited test {}] log.Log(170): Logging initialized @2295ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T07:27:07,176 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T07:27:07,238 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T07:27:07,257 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T07:27:07,257 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T07:27:07,258 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T07:27:07,277 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T07:27:07,280 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@77da9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/hadoop.log.dir/,AVAILABLE} 2024-11-26T07:27:07,281 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43ac9415{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T07:27:07,470 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@12264dc8{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/java.io.tmpdir/jetty-localhost-40117-hadoop-hdfs-3_4_1-tests_jar-_-any-11003727493957063337/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T07:27:07,476 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ce294cb{HTTP/1.1, (http/1.1)}{localhost:40117} 2024-11-26T07:27:07,476 INFO [Time-limited test {}] server.Server(415): Started @2683ms 2024-11-26T07:27:07,860 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T07:27:07,867 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T07:27:07,868 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T07:27:07,869 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T07:27:07,869 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T07:27:07,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@481eb1b6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/hadoop.log.dir/,AVAILABLE} 2024-11-26T07:27:07,871 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@991988e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T07:27:07,995 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3a1c1498{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/java.io.tmpdir/jetty-localhost-38395-hadoop-hdfs-3_4_1-tests_jar-_-any-8380257741255218239/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T07:27:07,996 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3badc187{HTTP/1.1, (http/1.1)}{localhost:38395} 2024-11-26T07:27:07,996 INFO [Time-limited test {}] server.Server(415): Started @3203ms 2024-11-26T07:27:08,051 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T07:27:08,178 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T07:27:08,183 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T07:27:08,185 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T07:27:08,185 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T07:27:08,185 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T07:27:08,186 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@157ee20f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/hadoop.log.dir/,AVAILABLE} 2024-11-26T07:27:08,186 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@424d969a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T07:27:08,307 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@41fe4ca9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/java.io.tmpdir/jetty-localhost-35975-hadoop-hdfs-3_4_1-tests_jar-_-any-4293536096284533063/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T07:27:08,308 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@74ee88d2{HTTP/1.1, (http/1.1)}{localhost:35975} 2024-11-26T07:27:08,308 INFO [Time-limited test {}] server.Server(415): Started @3515ms 2024-11-26T07:27:08,310 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T07:27:08,351 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T07:27:08,356 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T07:27:08,357 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T07:27:08,357 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T07:27:08,357 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T07:27:08,358 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d458b2a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/hadoop.log.dir/,AVAILABLE} 2024-11-26T07:27:08,359 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de3b13{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T07:27:08,489 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@31a39bd8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/java.io.tmpdir/jetty-localhost-39675-hadoop-hdfs-3_4_1-tests_jar-_-any-6550955990133256043/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T07:27:08,490 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5aa63b11{HTTP/1.1, (http/1.1)}{localhost:39675} 2024-11-26T07:27:08,491 INFO [Time-limited test {}] server.Server(415): Started @3698ms 2024-11-26T07:27:08,494 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T07:27:08,522 WARN [Thread-112 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data4/current/BP-568013986-172.17.0.2-1732606026764/current, will proceed with Du for space computation calculation, 2024-11-26T07:27:08,522 WARN [Thread-111 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data2/current/BP-568013986-172.17.0.2-1732606026764/current, will proceed with Du for space computation calculation, 2024-11-26T07:27:08,522 WARN [Thread-109 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data1/current/BP-568013986-172.17.0.2-1732606026764/current, will proceed with Du for space computation calculation, 2024-11-26T07:27:08,522 WARN [Thread-110 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data3/current/BP-568013986-172.17.0.2-1732606026764/current, will proceed with Du for space computation calculation, 2024-11-26T07:27:08,586 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T07:27:08,586 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T07:27:08,665 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data6/current/BP-568013986-172.17.0.2-1732606026764/current, will proceed with Du for space computation calculation, 2024-11-26T07:27:08,667 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data5/current/BP-568013986-172.17.0.2-1732606026764/current, will proceed with Du for space computation calculation, 2024-11-26T07:27:08,668 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x12bfa84dbfbaf518 with lease ID 0xeead779deab0d120: Processing first storage report for DS-036b023e-8ee6-41f7-ac95-88c88b652934 from datanode DatanodeRegistration(127.0.0.1:41787, datanodeUuid=62e50c5b-5dcc-4a7a-a637-006363e140ae, infoPort=41167, infoSecurePort=0, ipcPort=45073, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764) 2024-11-26T07:27:08,669 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12bfa84dbfbaf518 with lease ID 0xeead779deab0d120: from storage DS-036b023e-8ee6-41f7-ac95-88c88b652934 node DatanodeRegistration(127.0.0.1:41787, datanodeUuid=62e50c5b-5dcc-4a7a-a637-006363e140ae, infoPort=41167, infoSecurePort=0, ipcPort=45073, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T07:27:08,670 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x70ce9d7725b91795 with lease ID 0xeead779deab0d121: Processing first storage report for DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be from datanode DatanodeRegistration(127.0.0.1:40873, datanodeUuid=b95d11b6-f71c-42e0-8cb2-27719df50a40, infoPort=36293, infoSecurePort=0, ipcPort=36583, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764) 2024-11-26T07:27:08,670 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x70ce9d7725b91795 with lease ID 0xeead779deab0d121: from storage DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be node DatanodeRegistration(127.0.0.1:40873, datanodeUuid=b95d11b6-f71c-42e0-8cb2-27719df50a40, infoPort=36293, infoSecurePort=0, ipcPort=36583, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T07:27:08,670 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x70ce9d7725b91795 with lease ID 0xeead779deab0d121: Processing first storage report for DS-e8fac9cc-a0ef-42dc-8541-e1b61171f104 from datanode DatanodeRegistration(127.0.0.1:40873, datanodeUuid=b95d11b6-f71c-42e0-8cb2-27719df50a40, infoPort=36293, infoSecurePort=0, ipcPort=36583, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764) 2024-11-26T07:27:08,670 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x70ce9d7725b91795 with lease ID 0xeead779deab0d121: from storage DS-e8fac9cc-a0ef-42dc-8541-e1b61171f104 node DatanodeRegistration(127.0.0.1:40873, datanodeUuid=b95d11b6-f71c-42e0-8cb2-27719df50a40, infoPort=36293, infoSecurePort=0, ipcPort=36583, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T07:27:08,671 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x12bfa84dbfbaf518 with lease ID 0xeead779deab0d120: Processing first storage report for DS-28679e23-46c3-4cd6-8bd7-c49965de9794 from datanode DatanodeRegistration(127.0.0.1:41787, datanodeUuid=62e50c5b-5dcc-4a7a-a637-006363e140ae, infoPort=41167, infoSecurePort=0, ipcPort=45073, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764) 2024-11-26T07:27:08,671 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12bfa84dbfbaf518 with lease ID 0xeead779deab0d120: from storage DS-28679e23-46c3-4cd6-8bd7-c49965de9794 node DatanodeRegistration(127.0.0.1:41787, datanodeUuid=62e50c5b-5dcc-4a7a-a637-006363e140ae, infoPort=41167, infoSecurePort=0, ipcPort=45073, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T07:27:08,697 WARN [Thread-115 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T07:27:08,702 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8693c2b0fad95202 with lease ID 0xeead779deab0d122: Processing first storage report for DS-cc923b65-4ccd-4e0e-b363-2549e1aea740 from datanode DatanodeRegistration(127.0.0.1:42007, datanodeUuid=cef9386c-28c6-4e41-8f8f-519a1f9c1555, infoPort=46219, infoSecurePort=0, ipcPort=39871, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764) 2024-11-26T07:27:08,702 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8693c2b0fad95202 with lease ID 0xeead779deab0d122: from storage DS-cc923b65-4ccd-4e0e-b363-2549e1aea740 node DatanodeRegistration(127.0.0.1:42007, datanodeUuid=cef9386c-28c6-4e41-8f8f-519a1f9c1555, infoPort=46219, infoSecurePort=0, ipcPort=39871, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T07:27:08,702 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8693c2b0fad95202 with lease ID 0xeead779deab0d122: Processing first storage report for DS-fcf6a34f-5019-430c-a16b-f1af0d7f5a03 from datanode DatanodeRegistration(127.0.0.1:42007, datanodeUuid=cef9386c-28c6-4e41-8f8f-519a1f9c1555, infoPort=46219, infoSecurePort=0, ipcPort=39871, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764) 2024-11-26T07:27:08,703 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8693c2b0fad95202 with lease ID 0xeead779deab0d122: from storage DS-fcf6a34f-5019-430c-a16b-f1af0d7f5a03 node DatanodeRegistration(127.0.0.1:42007, datanodeUuid=cef9386c-28c6-4e41-8f8f-519a1f9c1555, infoPort=46219, infoSecurePort=0, ipcPort=39871, storageInfo=lv=-57;cid=testClusterID;nsid=1656477820;c=1732606026764), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T07:27:08,926 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78 2024-11-26T07:27:09,003 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/zookeeper_0, clientPort=57836, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T07:27:09,013 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57836 2024-11-26T07:27:09,025 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:09,028 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:09,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741825_1001 (size=7) 2024-11-26T07:27:09,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741825_1001 (size=7) 2024-11-26T07:27:09,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741825_1001 (size=7) 2024-11-26T07:27:09,662 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42 with version=8 2024-11-26T07:27:09,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/hbase-staging 2024-11-26T07:27:09,978 INFO [Time-limited test {}] client.ConnectionUtils(128): master/15ed1af4400d:0 server-side Connection retries=45 2024-11-26T07:27:09,989 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:09,989 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:09,994 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:27:09,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:09,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:27:10,130 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T07:27:10,191 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T07:27:10,200 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T07:27:10,204 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:27:10,231 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 19150 (auto-detected) 2024-11-26T07:27:10,233 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T07:27:10,252 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45945 2024-11-26T07:27:10,272 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45945 connecting to ZooKeeper ensemble=127.0.0.1:57836 2024-11-26T07:27:10,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459450x0, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:27:10,308 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45945-0x101532bd7460000 connected 2024-11-26T07:27:10,336 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:10,339 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:10,350 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:27:10,354 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42, hbase.cluster.distributed=false 2024-11-26T07:27:10,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:27:10,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45945 2024-11-26T07:27:10,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45945 2024-11-26T07:27:10,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45945 2024-11-26T07:27:10,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45945 2024-11-26T07:27:10,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45945 2024-11-26T07:27:10,490 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/15ed1af4400d:0 server-side Connection retries=45 2024-11-26T07:27:10,492 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:10,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:10,493 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:27:10,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:10,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:27:10,496 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T07:27:10,498 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:27:10,499 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42563 2024-11-26T07:27:10,501 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42563 connecting to ZooKeeper ensemble=127.0.0.1:57836 2024-11-26T07:27:10,502 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:10,504 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:10,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425630x0, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:27:10,510 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42563-0x101532bd7460001 connected 2024-11-26T07:27:10,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:27:10,514 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T07:27:10,522 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T07:27:10,525 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T07:27:10,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:27:10,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42563 2024-11-26T07:27:10,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42563 2024-11-26T07:27:10,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42563 2024-11-26T07:27:10,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42563 2024-11-26T07:27:10,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42563 2024-11-26T07:27:10,550 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/15ed1af4400d:0 server-side Connection retries=45 2024-11-26T07:27:10,550 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:10,550 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:10,551 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:27:10,551 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:10,551 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:27:10,552 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T07:27:10,552 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:27:10,553 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38495 2024-11-26T07:27:10,555 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38495 connecting to ZooKeeper ensemble=127.0.0.1:57836 2024-11-26T07:27:10,556 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:10,559 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:10,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:384950x0, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:27:10,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:384950x0, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:27:10,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38495-0x101532bd7460002 connected 2024-11-26T07:27:10,574 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T07:27:10,575 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T07:27:10,577 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T07:27:10,578 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:27:10,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38495 2024-11-26T07:27:10,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38495 2024-11-26T07:27:10,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38495 2024-11-26T07:27:10,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38495 2024-11-26T07:27:10,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38495 2024-11-26T07:27:10,598 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/15ed1af4400d:0 server-side Connection retries=45 2024-11-26T07:27:10,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:10,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:10,599 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:27:10,599 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:27:10,599 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:27:10,599 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T07:27:10,599 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:27:10,600 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42269 2024-11-26T07:27:10,602 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42269 connecting to ZooKeeper ensemble=127.0.0.1:57836 2024-11-26T07:27:10,603 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:10,605 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:10,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422690x0, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:27:10,612 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:422690x0, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:27:10,612 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42269-0x101532bd7460003 connected 2024-11-26T07:27:10,612 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T07:27:10,617 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T07:27:10,618 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T07:27:10,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:27:10,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42269 2024-11-26T07:27:10,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42269 2024-11-26T07:27:10,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42269 2024-11-26T07:27:10,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42269 2024-11-26T07:27:10,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42269 2024-11-26T07:27:10,641 DEBUG [M:0;15ed1af4400d:45945 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;15ed1af4400d:45945 2024-11-26T07:27:10,642 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/15ed1af4400d,45945,1732606029786 2024-11-26T07:27:10,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:27:10,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:27:10,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:27:10,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:27:10,655 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/15ed1af4400d,45945,1732606029786 2024-11-26T07:27:10,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:10,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T07:27:10,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T07:27:10,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T07:27:10,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:10,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:10,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:10,680 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T07:27:10,681 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/15ed1af4400d,45945,1732606029786 from backup master directory 2024-11-26T07:27:10,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:27:10,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:27:10,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/15ed1af4400d,45945,1732606029786 2024-11-26T07:27:10,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:27:10,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:27:10,686 WARN [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:27:10,686 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=15ed1af4400d,45945,1732606029786 2024-11-26T07:27:10,689 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T07:27:10,690 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T07:27:10,751 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/hbase.id] with ID: 566e8917-4cb2-4d05-8883-249e6a806b91 2024-11-26T07:27:10,751 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/.tmp/hbase.id 2024-11-26T07:27:10,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741826_1002 (size=42) 2024-11-26T07:27:10,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741826_1002 (size=42) 2024-11-26T07:27:10,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741826_1002 (size=42) 2024-11-26T07:27:10,769 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/.tmp/hbase.id]:[hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/hbase.id] 2024-11-26T07:27:10,815 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T07:27:10,820 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T07:27:10,840 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-26T07:27:10,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:10,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:10,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:10,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:10,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741827_1003 (size=196) 2024-11-26T07:27:10,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741827_1003 (size=196) 2024-11-26T07:27:10,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741827_1003 (size=196) 2024-11-26T07:27:10,882 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T07:27:10,884 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T07:27:10,890 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:10,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741828_1004 (size=1189) 2024-11-26T07:27:10,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741828_1004 (size=1189) 2024-11-26T07:27:10,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741828_1004 (size=1189) 2024-11-26T07:27:10,942 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store 2024-11-26T07:27:10,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741829_1005 (size=34) 2024-11-26T07:27:10,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741829_1005 (size=34) 2024-11-26T07:27:10,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741829_1005 (size=34) 2024-11-26T07:27:10,971 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T07:27:10,975 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:10,976 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T07:27:10,977 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:27:10,977 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:27:10,979 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T07:27:10,979 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:27:10,980 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:27:10,981 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732606030976Disabling compacts and flushes for region at 1732606030976Disabling writes for close at 1732606030979 (+3 ms)Writing region close event to WAL at 1732606030979Closed at 1732606030980 (+1 ms) 2024-11-26T07:27:10,984 WARN [master/15ed1af4400d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/.initializing 2024-11-26T07:27:10,984 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/WALs/15ed1af4400d,45945,1732606029786 2024-11-26T07:27:10,993 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:11,008 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=15ed1af4400d%2C45945%2C1732606029786, suffix=, logDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/WALs/15ed1af4400d,45945,1732606029786, archiveDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/oldWALs, maxLogs=10 2024-11-26T07:27:11,041 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/WALs/15ed1af4400d,45945,1732606029786/15ed1af4400d%2C45945%2C1732606029786.1732606031012, exclude list is [], retry=0 2024-11-26T07:27:11,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:11,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:11,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:11,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:11,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T07:27:11,107 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/WALs/15ed1af4400d,45945,1732606029786/15ed1af4400d%2C45945%2C1732606029786.1732606031012 2024-11-26T07:27:11,108 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:11,109 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:11,109 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:11,112 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,113 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,150 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T07:27:11,181 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:11,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:27:11,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T07:27:11,187 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:11,188 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:11,188 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T07:27:11,191 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:11,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:11,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,194 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T07:27:11,194 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:11,195 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:11,196 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,199 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,200 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,206 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,207 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,210 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T07:27:11,215 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:27:11,220 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:11,222 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67508098, jitterRate=0.005949050188064575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T07:27:11,228 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732606031126Initializing all the Stores at 1732606031128 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732606031128Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606031129 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606031129Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606031129Cleaning up temporary data from old regions at 1732606031207 (+78 ms)Region opened successfully at 1732606031228 (+21 ms) 2024-11-26T07:27:11,229 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T07:27:11,266 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4625cea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=15ed1af4400d/172.17.0.2:0 2024-11-26T07:27:11,300 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T07:27:11,311 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T07:27:11,312 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T07:27:11,315 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T07:27:11,316 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T07:27:11,321 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T07:27:11,321 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T07:27:11,346 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T07:27:11,354 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T07:27:11,358 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T07:27:11,361 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T07:27:11,362 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T07:27:11,364 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T07:27:11,366 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T07:27:11,369 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T07:27:11,371 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T07:27:11,372 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T07:27:11,373 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T07:27:11,392 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T07:27:11,395 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T07:27:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T07:27:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T07:27:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T07:27:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T07:27:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,402 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=15ed1af4400d,45945,1732606029786, sessionid=0x101532bd7460000, setting cluster-up flag (Was=false) 2024-11-26T07:27:11,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,422 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T07:27:11,424 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=15ed1af4400d,45945,1732606029786 2024-11-26T07:27:11,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:11,436 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T07:27:11,437 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=15ed1af4400d,45945,1732606029786 2024-11-26T07:27:11,444 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T07:27:11,517 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T07:27:11,528 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T07:27:11,529 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(746): ClusterId : 566e8917-4cb2-4d05-8883-249e6a806b91 2024-11-26T07:27:11,529 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(746): ClusterId : 566e8917-4cb2-4d05-8883-249e6a806b91 2024-11-26T07:27:11,529 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(746): ClusterId : 566e8917-4cb2-4d05-8883-249e6a806b91 2024-11-26T07:27:11,531 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T07:27:11,531 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T07:27:11,531 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T07:27:11,535 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T07:27:11,539 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T07:27:11,539 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T07:27:11,539 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T07:27:11,539 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T07:27:11,539 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T07:27:11,539 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T07:27:11,542 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T07:27:11,543 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T07:27:11,543 DEBUG [RS:1;15ed1af4400d:38495 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103a2b58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=15ed1af4400d/172.17.0.2:0 2024-11-26T07:27:11,543 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T07:27:11,543 DEBUG [RS:0;15ed1af4400d:42563 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16c5382e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=15ed1af4400d/172.17.0.2:0 2024-11-26T07:27:11,544 DEBUG [RS:2;15ed1af4400d:42269 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b30cf56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=15ed1af4400d/172.17.0.2:0 2024-11-26T07:27:11,541 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 15ed1af4400d,45945,1732606029786 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T07:27:11,549 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/15ed1af4400d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:27:11,550 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/15ed1af4400d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:27:11,550 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/15ed1af4400d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:27:11,550 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/15ed1af4400d:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:27:11,550 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/15ed1af4400d:0, corePoolSize=10, maxPoolSize=10 2024-11-26T07:27:11,551 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,551 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/15ed1af4400d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:27:11,551 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,561 DEBUG [RS:1;15ed1af4400d:38495 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;15ed1af4400d:38495 2024-11-26T07:27:11,564 DEBUG [RS:0;15ed1af4400d:42563 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;15ed1af4400d:42563 2024-11-26T07:27:11,565 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T07:27:11,565 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T07:27:11,565 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T07:27:11,565 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T07:27:11,565 DEBUG [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T07:27:11,565 DEBUG [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T07:27:11,568 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732606061567 2024-11-26T07:27:11,569 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(2659): reportForDuty to master=15ed1af4400d,45945,1732606029786 with port=38495, startcode=1732606030549 2024-11-26T07:27:11,569 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(2659): reportForDuty to master=15ed1af4400d,45945,1732606029786 with port=42563, startcode=1732606030451 2024-11-26T07:27:11,569 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T07:27:11,569 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T07:27:11,570 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T07:27:11,570 DEBUG [RS:2;15ed1af4400d:42269 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;15ed1af4400d:42269 2024-11-26T07:27:11,570 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T07:27:11,570 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T07:27:11,570 DEBUG [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T07:27:11,571 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T07:27:11,571 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(2659): reportForDuty to master=15ed1af4400d,45945,1732606029786 with port=42269, startcode=1732606030598 2024-11-26T07:27:11,575 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T07:27:11,575 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T07:27:11,576 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T07:27:11,576 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T07:27:11,577 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:11,577 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T07:27:11,582 DEBUG [RS:1;15ed1af4400d:38495 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T07:27:11,582 DEBUG [RS:2;15ed1af4400d:42269 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T07:27:11,582 DEBUG [RS:0;15ed1af4400d:42563 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T07:27:11,584 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,593 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T07:27:11,594 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T07:27:11,594 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T07:27:11,599 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T07:27:11,599 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T07:27:11,613 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/15ed1af4400d:0:becomeActiveMaster-HFileCleaner.large.0-1732606031600,5,FailOnTimeoutGroup] 2024-11-26T07:27:11,615 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/15ed1af4400d:0:becomeActiveMaster-HFileCleaner.small.0-1732606031614,5,FailOnTimeoutGroup] 2024-11-26T07:27:11,615 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,615 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T07:27:11,638 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,639 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741831_1007 (size=1321) 2024-11-26T07:27:11,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741831_1007 (size=1321) 2024-11-26T07:27:11,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741831_1007 (size=1321) 2024-11-26T07:27:11,644 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T07:27:11,644 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42 2024-11-26T07:27:11,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741832_1008 (size=32) 2024-11-26T07:27:11,664 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52759, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T07:27:11,664 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36001, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T07:27:11,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741832_1008 (size=32) 2024-11-26T07:27:11,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741832_1008 (size=32) 2024-11-26T07:27:11,666 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42929, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T07:27:11,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:11,672 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45945 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 15ed1af4400d,42563,1732606030451 2024-11-26T07:27:11,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T07:27:11,675 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45945 {}] master.ServerManager(517): Registering regionserver=15ed1af4400d,42563,1732606030451 2024-11-26T07:27:11,675 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T07:27:11,676 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:11,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:27:11,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T07:27:11,679 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T07:27:11,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:11,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:27:11,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T07:27:11,684 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T07:27:11,684 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:11,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:27:11,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T07:27:11,687 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T07:27:11,688 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:11,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:27:11,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T07:27:11,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740 2024-11-26T07:27:11,690 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45945 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 15ed1af4400d,42269,1732606030598 2024-11-26T07:27:11,690 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45945 {}] master.ServerManager(517): Registering regionserver=15ed1af4400d,42269,1732606030598 2024-11-26T07:27:11,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740 2024-11-26T07:27:11,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T07:27:11,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T07:27:11,694 DEBUG [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42 2024-11-26T07:27:11,695 DEBUG [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41363 2024-11-26T07:27:11,695 DEBUG [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T07:27:11,695 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T07:27:11,698 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45945 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 15ed1af4400d,38495,1732606030549 2024-11-26T07:27:11,698 DEBUG [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42 2024-11-26T07:27:11,698 DEBUG [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41363 2024-11-26T07:27:11,698 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45945 {}] master.ServerManager(517): Registering regionserver=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:11,698 DEBUG [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T07:27:11,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T07:27:11,702 DEBUG [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42 2024-11-26T07:27:11,702 DEBUG [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41363 2024-11-26T07:27:11,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:27:11,702 DEBUG [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T07:27:11,706 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:11,707 DEBUG [RS:0;15ed1af4400d:42563 {}] zookeeper.ZKUtil(111): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/15ed1af4400d,42563,1732606030451 2024-11-26T07:27:11,707 DEBUG [RS:2;15ed1af4400d:42269 {}] zookeeper.ZKUtil(111): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/15ed1af4400d,42269,1732606030598 2024-11-26T07:27:11,707 WARN [RS:2;15ed1af4400d:42269 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:27:11,707 WARN [RS:0;15ed1af4400d:42563 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:27:11,707 INFO [RS:2;15ed1af4400d:42269 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:11,707 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65436956, jitterRate=-0.024913370609283447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T07:27:11,707 DEBUG [RS:1;15ed1af4400d:38495 {}] zookeeper.ZKUtil(111): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/15ed1af4400d,38495,1732606030549 2024-11-26T07:27:11,707 DEBUG [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42269,1732606030598 2024-11-26T07:27:11,708 WARN [RS:1;15ed1af4400d:38495 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:27:11,708 INFO [RS:1;15ed1af4400d:38495 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:11,708 DEBUG [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,38495,1732606030549 2024-11-26T07:27:11,708 INFO [RS:0;15ed1af4400d:42563 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:11,709 DEBUG [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451 2024-11-26T07:27:11,709 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [15ed1af4400d,42563,1732606030451] 2024-11-26T07:27:11,709 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [15ed1af4400d,42269,1732606030598] 2024-11-26T07:27:11,709 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [15ed1af4400d,38495,1732606030549] 2024-11-26T07:27:11,711 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732606031670Initializing all the Stores at 1732606031672 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732606031672Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732606031672Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606031672Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732606031672Cleaning up temporary data from old regions at 1732606031694 (+22 ms)Region opened successfully at 1732606031711 (+17 ms) 2024-11-26T07:27:11,711 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T07:27:11,711 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T07:27:11,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T07:27:11,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T07:27:11,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T07:27:11,714 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T07:27:11,714 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732606031711Disabling compacts and flushes for region at 1732606031711Disabling writes for close at 1732606031712 (+1 ms)Writing region close event to WAL at 1732606031713 (+1 ms)Closed at 1732606031714 (+1 ms) 2024-11-26T07:27:11,718 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T07:27:11,718 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T07:27:11,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T07:27:11,736 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T07:27:11,739 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T07:27:11,743 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T07:27:11,743 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T07:27:11,743 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T07:27:11,760 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T07:27:11,760 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T07:27:11,760 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T07:27:11,766 INFO [RS:0;15ed1af4400d:42563 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T07:27:11,766 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,766 INFO [RS:2;15ed1af4400d:42269 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T07:27:11,766 INFO [RS:1;15ed1af4400d:38495 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T07:27:11,766 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,766 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,769 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T07:27:11,769 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T07:27:11,769 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T07:27:11,776 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T07:27:11,776 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T07:27:11,776 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T07:27:11,777 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,777 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,777 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,778 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,778 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:27:11,779 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:27:11,779 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:27:11,779 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,779 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,780 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,780 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,780 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,780 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,780 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/15ed1af4400d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:27:11,780 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/15ed1af4400d:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:27:11,780 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/15ed1af4400d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:27:11,780 DEBUG [RS:0;15ed1af4400d:42563 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/15ed1af4400d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:27:11,780 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/15ed1af4400d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:27:11,780 DEBUG [RS:1;15ed1af4400d:38495 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/15ed1af4400d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:27:11,780 DEBUG [RS:2;15ed1af4400d:42269 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/15ed1af4400d:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:27:11,787 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,787 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,787 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,787 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,788 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,788 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,42563,1732606030451-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:27:11,789 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,789 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,789 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,790 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,790 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,790 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,790 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,790 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,790 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,790 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,790 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,38495,1732606030549-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:27:11,790 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,42269,1732606030598-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:27:11,809 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T07:27:11,811 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T07:27:11,811 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,38495,1732606030549-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,811 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,42563,1732606030451-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,811 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,811 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,812 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.Replication(171): 15ed1af4400d,42563,1732606030451 started 2024-11-26T07:27:11,812 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.Replication(171): 15ed1af4400d,38495,1732606030549 started 2024-11-26T07:27:11,813 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T07:27:11,814 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,42269,1732606030598-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,814 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,814 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.Replication(171): 15ed1af4400d,42269,1732606030598 started 2024-11-26T07:27:11,832 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,832 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(1482): Serving as 15ed1af4400d,38495,1732606030549, RpcServer on 15ed1af4400d/172.17.0.2:38495, sessionid=0x101532bd7460002 2024-11-26T07:27:11,833 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T07:27:11,833 DEBUG [RS:1;15ed1af4400d:38495 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 15ed1af4400d,38495,1732606030549 2024-11-26T07:27:11,834 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '15ed1af4400d,38495,1732606030549' 2024-11-26T07:27:11,834 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T07:27:11,835 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T07:27:11,836 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T07:27:11,836 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T07:27:11,836 DEBUG [RS:1;15ed1af4400d:38495 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 15ed1af4400d,38495,1732606030549 2024-11-26T07:27:11,836 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '15ed1af4400d,38495,1732606030549' 2024-11-26T07:27:11,836 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T07:27:11,836 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,836 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:11,836 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(1482): Serving as 15ed1af4400d,42563,1732606030451, RpcServer on 15ed1af4400d/172.17.0.2:42563, sessionid=0x101532bd7460001 2024-11-26T07:27:11,836 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(1482): Serving as 15ed1af4400d,42269,1732606030598, RpcServer on 15ed1af4400d/172.17.0.2:42269, sessionid=0x101532bd7460003 2024-11-26T07:27:11,836 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T07:27:11,836 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T07:27:11,836 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T07:27:11,837 DEBUG [RS:2;15ed1af4400d:42269 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 15ed1af4400d,42269,1732606030598 2024-11-26T07:27:11,837 DEBUG [RS:0;15ed1af4400d:42563 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 15ed1af4400d,42563,1732606030451 2024-11-26T07:27:11,837 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '15ed1af4400d,42269,1732606030598' 2024-11-26T07:27:11,837 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '15ed1af4400d,42563,1732606030451' 2024-11-26T07:27:11,837 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T07:27:11,837 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T07:27:11,837 DEBUG [RS:1;15ed1af4400d:38495 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T07:27:11,837 INFO [RS:1;15ed1af4400d:38495 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T07:27:11,837 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T07:27:11,837 INFO [RS:1;15ed1af4400d:38495 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T07:27:11,837 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T07:27:11,838 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T07:27:11,838 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T07:27:11,838 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T07:27:11,838 DEBUG [RS:2;15ed1af4400d:42269 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 15ed1af4400d,42269,1732606030598 2024-11-26T07:27:11,838 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T07:27:11,838 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '15ed1af4400d,42269,1732606030598' 2024-11-26T07:27:11,838 DEBUG [RS:0;15ed1af4400d:42563 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 15ed1af4400d,42563,1732606030451 2024-11-26T07:27:11,838 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T07:27:11,838 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '15ed1af4400d,42563,1732606030451' 2024-11-26T07:27:11,838 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T07:27:11,839 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T07:27:11,839 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T07:27:11,839 DEBUG [RS:0;15ed1af4400d:42563 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T07:27:11,839 DEBUG [RS:2;15ed1af4400d:42269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T07:27:11,840 INFO [RS:0;15ed1af4400d:42563 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T07:27:11,840 INFO [RS:2;15ed1af4400d:42269 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T07:27:11,840 INFO [RS:0;15ed1af4400d:42563 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T07:27:11,840 INFO [RS:2;15ed1af4400d:42269 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T07:27:11,890 WARN [15ed1af4400d:45945 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T07:27:11,944 INFO [RS:2;15ed1af4400d:42269 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:11,944 INFO [RS:1;15ed1af4400d:38495 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:11,944 INFO [RS:0;15ed1af4400d:42563 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:11,948 INFO [RS:0;15ed1af4400d:42563 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=15ed1af4400d%2C42563%2C1732606030451, suffix=, logDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451, archiveDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/oldWALs, maxLogs=32 2024-11-26T07:27:11,948 INFO [RS:1;15ed1af4400d:38495 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=15ed1af4400d%2C38495%2C1732606030549, suffix=, logDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,38495,1732606030549, archiveDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/oldWALs, maxLogs=32 2024-11-26T07:27:11,948 INFO [RS:2;15ed1af4400d:42269 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=15ed1af4400d%2C42269%2C1732606030598, suffix=, logDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42269,1732606030598, archiveDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/oldWALs, maxLogs=32 2024-11-26T07:27:11,969 DEBUG [RS:0;15ed1af4400d:42563 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451/15ed1af4400d%2C42563%2C1732606030451.1732606031953, exclude list is [], retry=0 2024-11-26T07:27:11,969 DEBUG [RS:2;15ed1af4400d:42269 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42269,1732606030598/15ed1af4400d%2C42269%2C1732606030598.1732606031953, exclude list is [], retry=0 2024-11-26T07:27:11,973 DEBUG [RS:1;15ed1af4400d:38495 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,38495,1732606030549/15ed1af4400d%2C38495%2C1732606030549.1732606031953, exclude list is [], retry=0 2024-11-26T07:27:11,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:11,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:11,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:11,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:11,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:11,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:11,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:11,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:11,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:11,989 INFO [RS:2;15ed1af4400d:42269 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42269,1732606030598/15ed1af4400d%2C42269%2C1732606030598.1732606031953 2024-11-26T07:27:11,989 INFO [RS:0;15ed1af4400d:42563 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451/15ed1af4400d%2C42563%2C1732606030451.1732606031953 2024-11-26T07:27:11,993 DEBUG [RS:2;15ed1af4400d:42269 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:46219:46219)] 2024-11-26T07:27:11,993 DEBUG [RS:0;15ed1af4400d:42563 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219)] 2024-11-26T07:27:11,994 INFO [RS:1;15ed1af4400d:38495 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,38495,1732606030549/15ed1af4400d%2C38495%2C1732606030549.1732606031953 2024-11-26T07:27:11,997 DEBUG [RS:1;15ed1af4400d:38495 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:12,143 DEBUG [15ed1af4400d:45945 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-26T07:27:12,151 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(204): Hosts are {15ed1af4400d=0} racks are {/default-rack=0} 2024-11-26T07:27:12,157 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T07:27:12,157 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T07:27:12,157 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T07:27:12,157 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T07:27:12,158 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T07:27:12,158 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T07:27:12,158 INFO [15ed1af4400d:45945 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T07:27:12,158 INFO [15ed1af4400d:45945 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T07:27:12,158 INFO [15ed1af4400d:45945 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T07:27:12,158 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T07:27:12,165 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:12,170 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 15ed1af4400d,38495,1732606030549, state=OPENING 2024-11-26T07:27:12,174 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T07:27:12,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:12,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:12,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:12,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:12,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:27:12,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:27:12,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:27:12,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:27:12,178 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T07:27:12,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=15ed1af4400d,38495,1732606030549}] 2024-11-26T07:27:12,355 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T07:27:12,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57395, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T07:27:12,370 INFO [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T07:27:12,371 INFO [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:12,371 INFO [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T07:27:12,374 INFO [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=15ed1af4400d%2C38495%2C1732606030549.meta, suffix=.meta, logDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,38495,1732606030549, archiveDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/oldWALs, maxLogs=32 2024-11-26T07:27:12,390 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,38495,1732606030549/15ed1af4400d%2C38495%2C1732606030549.meta.1732606032376.meta, exclude list is [], retry=0 2024-11-26T07:27:12,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:12,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:12,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:12,397 INFO [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,38495,1732606030549/15ed1af4400d%2C38495%2C1732606030549.meta.1732606032376.meta 2024-11-26T07:27:12,397 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:12,397 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:12,399 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T07:27:12,402 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T07:27:12,406 INFO [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T07:27:12,410 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T07:27:12,411 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:12,411 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T07:27:12,411 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T07:27:12,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T07:27:12,416 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T07:27:12,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:12,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:27:12,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T07:27:12,418 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T07:27:12,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:12,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:27:12,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T07:27:12,421 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T07:27:12,421 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:12,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:27:12,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T07:27:12,423 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T07:27:12,423 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:12,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:27:12,424 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T07:27:12,425 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740 2024-11-26T07:27:12,428 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740 2024-11-26T07:27:12,430 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T07:27:12,430 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T07:27:12,431 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T07:27:12,433 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T07:27:12,435 INFO [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62337311, jitterRate=-0.07110168039798737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T07:27:12,435 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T07:27:12,436 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732606032412Writing region info on filesystem at 1732606032412Initializing all the Stores at 1732606032414 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732606032414Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732606032414Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606032414Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732606032414Cleaning up temporary data from old regions at 1732606032430 (+16 ms)Running coprocessor post-open hooks at 1732606032435 (+5 ms)Region opened successfully at 1732606032436 (+1 ms) 2024-11-26T07:27:12,444 INFO [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732606032347 2024-11-26T07:27:12,455 DEBUG [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T07:27:12,456 INFO [RS_OPEN_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T07:27:12,457 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:12,459 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 15ed1af4400d,38495,1732606030549, state=OPEN 2024-11-26T07:27:12,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:27:12,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:27:12,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:27:12,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:27:12,463 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:27:12,463 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:27:12,463 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:27:12,463 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:27:12,463 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:12,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T07:27:12,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=15ed1af4400d,38495,1732606030549 in 283 msec 2024-11-26T07:27:12,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T07:27:12,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 746 msec 2024-11-26T07:27:12,478 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T07:27:12,478 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T07:27:12,500 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T07:27:12,501 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15ed1af4400d,38495,1732606030549, seqNum=-1] 2024-11-26T07:27:12,522 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T07:27:12,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48153, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T07:27:12,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0760 sec 2024-11-26T07:27:12,549 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732606032549, completionTime=-1 2024-11-26T07:27:12,551 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-26T07:27:12,552 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T07:27:12,577 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-26T07:27:12,577 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732606092577 2024-11-26T07:27:12,577 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732606152577 2024-11-26T07:27:12,577 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-26T07:27:12,579 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T07:27:12,585 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,45945,1732606029786-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:12,586 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,45945,1732606029786-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:12,586 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,45945,1732606029786-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:12,587 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-15ed1af4400d:45945, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:12,588 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:12,588 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:12,595 DEBUG [master/15ed1af4400d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T07:27:12,616 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.930sec 2024-11-26T07:27:12,618 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T07:27:12,619 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T07:27:12,620 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T07:27:12,621 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T07:27:12,621 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T07:27:12,622 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,45945,1732606029786-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:27:12,622 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,45945,1732606029786-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T07:27:12,627 DEBUG [master/15ed1af4400d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T07:27:12,628 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T07:27:12,628 INFO [master/15ed1af4400d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15ed1af4400d,45945,1732606029786-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:12,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@304fe8ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T07:27:12,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 15ed1af4400d,45945,-1 for getting cluster id 2024-11-26T07:27:12,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T07:27:12,655 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '566e8917-4cb2-4d05-8883-249e6a806b91' 2024-11-26T07:27:12,657 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T07:27:12,657 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "566e8917-4cb2-4d05-8883-249e6a806b91" 2024-11-26T07:27:12,658 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428a511d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T07:27:12,658 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [15ed1af4400d,45945,-1] 2024-11-26T07:27:12,660 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T07:27:12,663 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:27:12,663 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T07:27:12,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180d5a68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T07:27:12,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T07:27:12,673 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15ed1af4400d,38495,1732606030549, seqNum=-1] 2024-11-26T07:27:12,674 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T07:27:12,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T07:27:12,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=15ed1af4400d,45945,1732606029786 2024-11-26T07:27:12,698 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:41363/hbase 2024-11-26T07:27:12,714 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=8440 2024-11-26T07:27:12,733 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:12,736 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:12,737 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:12,741 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72559631, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/hregion-72559631, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:12,758 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72559631/hregion-72559631.1732606032743, exclude list is [], retry=0 2024-11-26T07:27:12,763 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:12,763 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:12,763 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:12,767 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72559631/hregion-72559631.1732606032743 2024-11-26T07:27:12,767 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:46219:46219)] 2024-11-26T07:27:12,767 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 740ff8eeba4c318a2b2ed54adeada6fd, NAME => 'testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/hbase 2024-11-26T07:27:12,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741838_1014 (size=64) 2024-11-26T07:27:12,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741838_1014 (size=64) 2024-11-26T07:27:12,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741838_1014 (size=64) 2024-11-26T07:27:12,785 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:12,787 INFO [StoreOpener-740ff8eeba4c318a2b2ed54adeada6fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:12,789 INFO [StoreOpener-740ff8eeba4c318a2b2ed54adeada6fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 740ff8eeba4c318a2b2ed54adeada6fd columnFamilyName a 2024-11-26T07:27:12,790 DEBUG [StoreOpener-740ff8eeba4c318a2b2ed54adeada6fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:12,790 INFO [StoreOpener-740ff8eeba4c318a2b2ed54adeada6fd-1 {}] regionserver.HStore(327): Store=740ff8eeba4c318a2b2ed54adeada6fd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:12,791 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:12,792 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:12,793 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:12,794 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:12,794 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:12,797 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:12,801 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:12,801 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 740ff8eeba4c318a2b2ed54adeada6fd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62295581, jitterRate=-0.07172350585460663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:27:12,803 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 740ff8eeba4c318a2b2ed54adeada6fd: Writing region info on filesystem at 1732606032785Initializing all the Stores at 1732606032786 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606032787 (+1 ms)Cleaning up temporary data from old regions at 1732606032794 (+7 ms)Region opened successfully at 1732606032803 (+9 ms) 2024-11-26T07:27:12,803 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 740ff8eeba4c318a2b2ed54adeada6fd, disabling compactions & flushes 2024-11-26T07:27:12,804 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd. 2024-11-26T07:27:12,804 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd. 2024-11-26T07:27:12,804 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd. after waiting 0 ms 2024-11-26T07:27:12,804 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd. 2024-11-26T07:27:12,804 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd. 2024-11-26T07:27:12,804 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 740ff8eeba4c318a2b2ed54adeada6fd: Waiting for close lock at 1732606032803Disabling compacts and flushes for region at 1732606032803Disabling writes for close at 1732606032804 (+1 ms)Writing region close event to WAL at 1732606032804Closed at 1732606032804 2024-11-26T07:27:12,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741837_1013 (size=93) 2024-11-26T07:27:12,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741837_1013 (size=93) 2024-11-26T07:27:12,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741837_1013 (size=93) 2024-11-26T07:27:12,817 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:12,817 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72559631:(num 1732606032743) 2024-11-26T07:27:12,819 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T07:27:12,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741839_1015 (size=276) 2024-11-26T07:27:12,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741839_1015 (size=276) 2024-11-26T07:27:12,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741839_1015 (size=276) 2024-11-26T07:27:12,840 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T07:27:12,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741840_1016 (size=230) 2024-11-26T07:27:12,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741840_1016 (size=230) 2024-11-26T07:27:12,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741840_1016 (size=230) 2024-11-26T07:27:12,876 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1, size=276 (276bytes) 2024-11-26T07:27:12,876 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-26T07:27:12,877 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-26T07:27:12,877 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1 2024-11-26T07:27:12,883 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1 after 3ms 2024-11-26T07:27:12,890 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:12,891 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1 took 17ms 2024-11-26T07:27:12,895 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1 so closing down 2024-11-26T07:27:12,899 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-26T07:27:12,900 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:12,901 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000001-wal-1.temp 2024-11-26T07:27:12,902 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:12,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741841_1017 (size=276) 2024-11-26T07:27:12,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741841_1017 (size=276) 2024-11-26T07:27:12,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741841_1017 (size=276) 2024-11-26T07:27:12,914 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:12,917 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002 2024-11-26T07:27:12,921 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-26T07:27:12,921 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1, journal: Splitting hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1, size=276 (276bytes) at 1732606032876Finishing writing output for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1 so closing down at 1732606032895 (+19 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000001-wal-1.temp at 1732606032901 (+6 ms)3 split writer threads finished at 1732606032902 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732606032914 (+12 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002 at 1732606032917 (+3 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732606032921 (+4 ms) 2024-11-26T07:27:12,941 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2, size=230 (230bytes) 2024-11-26T07:27:12,941 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2 2024-11-26T07:27:12,942 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2 after 1ms 2024-11-26T07:27:12,946 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:12,947 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2 took 6ms 2024-11-26T07:27:12,949 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2 so closing down 2024-11-26T07:27:12,949 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:12,952 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-26T07:27:12,954 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002-wal-2.temp 2024-11-26T07:27:12,954 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:12,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741842_1018 (size=230) 2024-11-26T07:27:12,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741842_1018 (size=230) 2024-11-26T07:27:12,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741842_1018 (size=230) 2024-11-26T07:27:12,966 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:12,971 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:12,973 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-11-26T07:27:12,975 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-26T07:27:12,975 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2, journal: Splitting hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2, size=230 (230bytes) at 1732606032941Finishing writing output for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2 so closing down at 1732606032949 (+8 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002-wal-2.temp at 1732606032954 (+5 ms)3 split writer threads finished at 1732606032954Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732606032966 (+12 ms)Processed 1 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732606032975 (+9 ms) 2024-11-26T07:27:12,975 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:12,977 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:12,993 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal.1732606032979, exclude list is [], retry=0 2024-11-26T07:27:12,997 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:12,998 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:12,998 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:13,001 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal.1732606032979 2024-11-26T07:27:13,001 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219)] 2024-11-26T07:27:13,001 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 740ff8eeba4c318a2b2ed54adeada6fd, NAME => 'testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:13,002 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:13,002 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:13,002 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:13,004 INFO [StoreOpener-740ff8eeba4c318a2b2ed54adeada6fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:13,006 INFO [StoreOpener-740ff8eeba4c318a2b2ed54adeada6fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 740ff8eeba4c318a2b2ed54adeada6fd columnFamilyName a 2024-11-26T07:27:13,006 DEBUG [StoreOpener-740ff8eeba4c318a2b2ed54adeada6fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:13,007 INFO [StoreOpener-740ff8eeba4c318a2b2ed54adeada6fd-1 {}] regionserver.HStore(327): Store=740ff8eeba4c318a2b2ed54adeada6fd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:13,007 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:13,008 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:13,011 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:13,012 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002 2024-11-26T07:27:13,016 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:13,022 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002 2024-11-26T07:27:13,026 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 740ff8eeba4c318a2b2ed54adeada6fd 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-26T07:27:13,081 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/.tmp/a/72213686bae5450bac8c4ed91e2af4d9 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732606032817/Put/seqid=0 2024-11-26T07:27:13,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741844_1020 (size=5170) 2024-11-26T07:27:13,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741844_1020 (size=5170) 2024-11-26T07:27:13,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741844_1020 (size=5170) 2024-11-26T07:27:13,096 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/.tmp/a/72213686bae5450bac8c4ed91e2af4d9 2024-11-26T07:27:13,180 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/.tmp/a/72213686bae5450bac8c4ed91e2af4d9 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/a/72213686bae5450bac8c4ed91e2af4d9 2024-11-26T07:27:13,189 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/a/72213686bae5450bac8c4ed91e2af4d9, entries=2, sequenceid=2, filesize=5.0 K 2024-11-26T07:27:13,194 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 740ff8eeba4c318a2b2ed54adeada6fd in 167ms, sequenceid=2, compaction requested=false; wal=null 2024-11-26T07:27:13,195 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/0000000000000000002 2024-11-26T07:27:13,196 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:13,196 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:13,198 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 740ff8eeba4c318a2b2ed54adeada6fd 2024-11-26T07:27:13,201 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/740ff8eeba4c318a2b2ed54adeada6fd/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-26T07:27:13,203 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 740ff8eeba4c318a2b2ed54adeada6fd; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69108006, jitterRate=0.02978953719139099}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:27:13,204 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 740ff8eeba4c318a2b2ed54adeada6fd: Writing region info on filesystem at 1732606033002Initializing all the Stores at 1732606033003 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606033003Obtaining lock to block concurrent updates at 1732606033026 (+23 ms)Preparing flush snapshotting stores in 740ff8eeba4c318a2b2ed54adeada6fd at 1732606033026Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732606033029 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732606032734.740ff8eeba4c318a2b2ed54adeada6fd. at 1732606033029Flushing 740ff8eeba4c318a2b2ed54adeada6fd/a: creating writer at 1732606033030 (+1 ms)Flushing 740ff8eeba4c318a2b2ed54adeada6fd/a: appending metadata at 1732606033070 (+40 ms)Flushing 740ff8eeba4c318a2b2ed54adeada6fd/a: closing flushed file at 1732606033073 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62cd414: reopening flushed file at 1732606033178 (+105 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 740ff8eeba4c318a2b2ed54adeada6fd in 167ms, sequenceid=2, compaction requested=false; wal=null at 1732606033194 (+16 ms)Cleaning up temporary data from old regions at 1732606033196 (+2 ms)Region opened successfully at 1732606033204 (+8 ms) 2024-11-26T07:27:13,231 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=376 (was 364) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:41363/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:45398 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:42046 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:57614 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:57428 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:42070 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:45564 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=701 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 220), ProcessCount=11 (was 11), AvailableMemoryMB=8357 (was 8440) 2024-11-26T07:27:13,241 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=701, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=8357 2024-11-26T07:27:13,259 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:13,262 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:13,263 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:13,267 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-15531664, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/hregion-15531664, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:13,280 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-15531664/hregion-15531664.1732606033267, exclude list is [], retry=0 2024-11-26T07:27:13,284 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:13,284 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:13,284 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:13,287 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-15531664/hregion-15531664.1732606033267 2024-11-26T07:27:13,288 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:13,288 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3a6d2f18e80707a3951a7bb150d59aa4, NAME => 'testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/hbase 2024-11-26T07:27:13,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741846_1022 (size=64) 2024-11-26T07:27:13,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741846_1022 (size=64) 2024-11-26T07:27:13,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741846_1022 (size=64) 2024-11-26T07:27:13,300 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:13,301 INFO [StoreOpener-3a6d2f18e80707a3951a7bb150d59aa4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,303 INFO [StoreOpener-3a6d2f18e80707a3951a7bb150d59aa4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a6d2f18e80707a3951a7bb150d59aa4 columnFamilyName a 2024-11-26T07:27:13,303 DEBUG [StoreOpener-3a6d2f18e80707a3951a7bb150d59aa4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:13,304 INFO [StoreOpener-3a6d2f18e80707a3951a7bb150d59aa4-1 {}] regionserver.HStore(327): Store=3a6d2f18e80707a3951a7bb150d59aa4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:13,304 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,306 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,307 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,307 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,309 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,313 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:13,313 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3a6d2f18e80707a3951a7bb150d59aa4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62196097, jitterRate=-0.07320593297481537}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:27:13,313 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3a6d2f18e80707a3951a7bb150d59aa4: Writing region info on filesystem at 1732606033300Initializing all the Stores at 1732606033301 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606033301Cleaning up temporary data from old regions at 1732606033307 (+6 ms)Region opened successfully at 1732606033313 (+6 ms) 2024-11-26T07:27:13,314 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3a6d2f18e80707a3951a7bb150d59aa4, disabling compactions & flushes 2024-11-26T07:27:13,314 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4. 2024-11-26T07:27:13,314 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4. 2024-11-26T07:27:13,314 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4. after waiting 0 ms 2024-11-26T07:27:13,314 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4. 2024-11-26T07:27:13,314 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4. 2024-11-26T07:27:13,314 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3a6d2f18e80707a3951a7bb150d59aa4: Waiting for close lock at 1732606033314Disabling compacts and flushes for region at 1732606033314Disabling writes for close at 1732606033314Writing region close event to WAL at 1732606033314Closed at 1732606033314 2024-11-26T07:27:13,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741845_1021 (size=93) 2024-11-26T07:27:13,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741845_1021 (size=93) 2024-11-26T07:27:13,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741845_1021 (size=93) 2024-11-26T07:27:13,322 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:13,322 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-15531664:(num 1732606033267) 2024-11-26T07:27:13,323 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T07:27:13,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741847_1023 (size=276) 2024-11-26T07:27:13,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741847_1023 (size=276) 2024-11-26T07:27:13,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741847_1023 (size=276) 2024-11-26T07:27:13,337 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T07:27:13,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741848_1024 (size=230) 2024-11-26T07:27:13,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741848_1024 (size=230) 2024-11-26T07:27:13,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741848_1024 (size=230) 2024-11-26T07:27:13,368 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2, size=230 (230bytes) 2024-11-26T07:27:13,368 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2 2024-11-26T07:27:13,369 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2 after 1ms 2024-11-26T07:27:13,373 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:13,373 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2 took 5ms 2024-11-26T07:27:13,375 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2 so closing down 2024-11-26T07:27:13,375 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:13,378 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-26T07:27:13,379 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002-wal-2.temp 2024-11-26T07:27:13,379 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:13,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741849_1025 (size=230) 2024-11-26T07:27:13,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741849_1025 (size=230) 2024-11-26T07:27:13,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741849_1025 (size=230) 2024-11-26T07:27:13,388 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:13,390 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002 2024-11-26T07:27:13,390 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-26T07:27:13,390 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2, journal: Splitting hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2, size=230 (230bytes) at 1732606033368Finishing writing output for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2 so closing down at 1732606033375 (+7 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002-wal-2.temp at 1732606033379 (+4 ms)3 split writer threads finished at 1732606033379Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732606033388 (+9 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002 at 1732606033390 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732606033390 2024-11-26T07:27:13,405 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1, size=276 (276bytes) 2024-11-26T07:27:13,405 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1 2024-11-26T07:27:13,405 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1 after 0ms 2024-11-26T07:27:13,409 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:13,409 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1 took 4ms 2024-11-26T07:27:13,411 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1 so closing down 2024-11-26T07:27:13,411 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:13,413 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-26T07:27:13,415 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000001-wal-1.temp 2024-11-26T07:27:13,415 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:13,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741850_1026 (size=276) 2024-11-26T07:27:13,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741850_1026 (size=276) 2024-11-26T07:27:13,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741850_1026 (size=276) 2024-11-26T07:27:13,423 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:13,427 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:13,429 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002, length=230 2024-11-26T07:27:13,431 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002 2024-11-26T07:27:13,432 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-26T07:27:13,432 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1, journal: Splitting hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1, size=276 (276bytes) at 1732606033405Finishing writing output for hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1 so closing down at 1732606033411 (+6 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000001-wal-1.temp at 1732606033415 (+4 ms)3 split writer threads finished at 1732606033415Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732606033423 (+8 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002 at 1732606033431 (+8 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732606033432 (+1 ms) 2024-11-26T07:27:13,432 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:13,434 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:13,448 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal.1732606033435, exclude list is [], retry=0 2024-11-26T07:27:13,451 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:13,451 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:13,452 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:13,454 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal.1732606033435 2024-11-26T07:27:13,454 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:13,454 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a6d2f18e80707a3951a7bb150d59aa4, NAME => 'testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:13,455 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:13,455 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,455 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,457 INFO [StoreOpener-3a6d2f18e80707a3951a7bb150d59aa4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,459 INFO [StoreOpener-3a6d2f18e80707a3951a7bb150d59aa4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a6d2f18e80707a3951a7bb150d59aa4 columnFamilyName a 2024-11-26T07:27:13,460 DEBUG [StoreOpener-3a6d2f18e80707a3951a7bb150d59aa4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:13,460 INFO [StoreOpener-3a6d2f18e80707a3951a7bb150d59aa4-1 {}] regionserver.HStore(327): Store=3a6d2f18e80707a3951a7bb150d59aa4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:13,460 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,461 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,463 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,464 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002 2024-11-26T07:27:13,467 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:13,468 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002 2024-11-26T07:27:13,468 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3a6d2f18e80707a3951a7bb150d59aa4 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-26T07:27:13,485 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/.tmp/a/2726a43c73c14c7b8a0c4e2e04a35274 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732606033322/Put/seqid=0 2024-11-26T07:27:13,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741852_1028 (size=5170) 2024-11-26T07:27:13,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741852_1028 (size=5170) 2024-11-26T07:27:13,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741852_1028 (size=5170) 2024-11-26T07:27:13,495 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/.tmp/a/2726a43c73c14c7b8a0c4e2e04a35274 2024-11-26T07:27:13,503 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/.tmp/a/2726a43c73c14c7b8a0c4e2e04a35274 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/a/2726a43c73c14c7b8a0c4e2e04a35274 2024-11-26T07:27:13,512 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/a/2726a43c73c14c7b8a0c4e2e04a35274, entries=2, sequenceid=2, filesize=5.0 K 2024-11-26T07:27:13,512 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3a6d2f18e80707a3951a7bb150d59aa4 in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-11-26T07:27:13,513 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/0000000000000000002 2024-11-26T07:27:13,514 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,514 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,517 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3a6d2f18e80707a3951a7bb150d59aa4 2024-11-26T07:27:13,520 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/3a6d2f18e80707a3951a7bb150d59aa4/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-26T07:27:13,521 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3a6d2f18e80707a3951a7bb150d59aa4; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61655003, jitterRate=-0.08126886188983917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:27:13,521 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3a6d2f18e80707a3951a7bb150d59aa4: Writing region info on filesystem at 1732606033455Initializing all the Stores at 1732606033456 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606033456Obtaining lock to block concurrent updates at 1732606033468 (+12 ms)Preparing flush snapshotting stores in 3a6d2f18e80707a3951a7bb150d59aa4 at 1732606033468Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732606033468Flushing stores of testReplayEditsWrittenIntoWAL,,1732606033260.3a6d2f18e80707a3951a7bb150d59aa4. at 1732606033468Flushing 3a6d2f18e80707a3951a7bb150d59aa4/a: creating writer at 1732606033469 (+1 ms)Flushing 3a6d2f18e80707a3951a7bb150d59aa4/a: appending metadata at 1732606033484 (+15 ms)Flushing 3a6d2f18e80707a3951a7bb150d59aa4/a: closing flushed file at 1732606033484Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53c6392: reopening flushed file at 1732606033502 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3a6d2f18e80707a3951a7bb150d59aa4 in 44ms, sequenceid=2, compaction requested=false; wal=null at 1732606033512 (+10 ms)Cleaning up temporary data from old regions at 1732606033514 (+2 ms)Region opened successfully at 1732606033521 (+7 ms) 2024-11-26T07:27:13,541 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:57694 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:45644 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:42128 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:42070 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:45398 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:57428 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 701) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=242 (was 220) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8349 (was 8357) 2024-11-26T07:27:13,552 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=8349 2024-11-26T07:27:13,568 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:13,570 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:13,571 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:13,574 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-22057706, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/hregion-22057706, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:13,587 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-22057706/hregion-22057706.1732606033575, exclude list is [], retry=0 2024-11-26T07:27:13,591 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:13,591 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:13,591 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:13,594 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-22057706/hregion-22057706.1732606033575 2024-11-26T07:27:13,594 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:46219:46219)] 2024-11-26T07:27:13,594 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6a7e797a660052076a77697ee9ce9842, NAME => 'testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/hbase 2024-11-26T07:27:13,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741854_1030 (size=64) 2024-11-26T07:27:13,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741854_1030 (size=64) 2024-11-26T07:27:13,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741854_1030 (size=64) 2024-11-26T07:27:13,605 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:13,606 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:13,608 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a7e797a660052076a77697ee9ce9842 columnFamilyName a 2024-11-26T07:27:13,608 DEBUG [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:13,609 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(327): Store=6a7e797a660052076a77697ee9ce9842/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:13,609 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:13,611 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a7e797a660052076a77697ee9ce9842 columnFamilyName b 2024-11-26T07:27:13,611 DEBUG [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:13,611 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(327): Store=6a7e797a660052076a77697ee9ce9842/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:13,611 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:13,613 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a7e797a660052076a77697ee9ce9842 columnFamilyName c 2024-11-26T07:27:13,613 DEBUG [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:13,613 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(327): Store=6a7e797a660052076a77697ee9ce9842/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:13,614 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:13,614 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:13,615 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:13,616 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:13,616 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:13,617 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:13,618 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:13,621 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:13,621 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a7e797a660052076a77697ee9ce9842; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75349801, jitterRate=0.12279953062534332}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:13,622 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a7e797a660052076a77697ee9ce9842: Writing region info on filesystem at 1732606033605Initializing all the Stores at 1732606033606 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606033606Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606033606Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606033606Cleaning up temporary data from old regions at 1732606033616 (+10 ms)Region opened successfully at 1732606033621 (+5 ms) 2024-11-26T07:27:13,622 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a7e797a660052076a77697ee9ce9842, disabling compactions & flushes 2024-11-26T07:27:13,622 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. 2024-11-26T07:27:13,622 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. 2024-11-26T07:27:13,622 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. after waiting 0 ms 2024-11-26T07:27:13,622 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. 2024-11-26T07:27:13,622 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. 2024-11-26T07:27:13,622 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a7e797a660052076a77697ee9ce9842: Waiting for close lock at 1732606033622Disabling compacts and flushes for region at 1732606033622Disabling writes for close at 1732606033622Writing region close event to WAL at 1732606033622Closed at 1732606033622 2024-11-26T07:27:13,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741853_1029 (size=93) 2024-11-26T07:27:13,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741853_1029 (size=93) 2024-11-26T07:27:13,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741853_1029 (size=93) 2024-11-26T07:27:13,629 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:13,629 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-22057706:(num 1732606033575) 2024-11-26T07:27:13,629 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:13,631 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:13,646 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632, exclude list is [], retry=0 2024-11-26T07:27:13,649 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:13,650 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:13,650 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:13,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 2024-11-26T07:27:13,653 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:13,860 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632, size=0 (0bytes) 2024-11-26T07:27:13,860 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 might be still open, length is 0 2024-11-26T07:27:13,860 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 2024-11-26T07:27:13,861 WARN [IPC Server handler 2 on default port 41363 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-26T07:27:13,862 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 after 2ms 2024-11-26T07:27:14,719 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:57736 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40873:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57736 dst: /127.0.0.1:40873 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40873 remote=/127.0.0.1:57736]. Total timeout mills is 60000, 59108 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:14,720 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:45692 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:42007:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45692 dst: /127.0.0.1:42007 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:14,720 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:42164 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42164 dst: /127.0.0.1:41787 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:14,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741855_1032 (size=200589) 2024-11-26T07:27:14,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741855_1032 (size=200589) 2024-11-26T07:27:14,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741855_1032 (size=200589) 2024-11-26T07:27:17,863 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 after 4003ms 2024-11-26T07:27:17,867 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:17,869 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 took 4009ms 2024-11-26T07:27:17,873 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732606033632.temp 2024-11-26T07:27:17,877 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000000001-wal.1732606033632.temp 2024-11-26T07:27:17,915 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T07:27:17,986 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T07:27:17,993 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632; continuing. 2024-11-26T07:27:17,993 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 so closing down 2024-11-26T07:27:17,993 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:17,994 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:17,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741856_1033 (size=200597) 2024-11-26T07:27:17,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741856_1033 (size=200597) 2024-11-26T07:27:17,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741856_1033 (size=200597) 2024-11-26T07:27:17,998 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000000001-wal.1732606033632.temp (wrote 3002 edits, skipped 0 edits in 46 ms) 2024-11-26T07:27:18,000 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000000001-wal.1732606033632.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000003002 2024-11-26T07:27:18,000 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 131 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T07:27:18,000 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632, journal: Splitting hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632, size=0 (0bytes) at 1732606033860Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000000001-wal.1732606033632.temp at 1732606037877 (+4017 ms)Split 1024 edits, skipped 0 edits. at 1732606037932 (+55 ms)Split 2048 edits, skipped 0 edits. at 1732606037966 (+34 ms)Finishing writing output for hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 so closing down at 1732606037993 (+27 ms)3 split writer threads finished at 1732606037994 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000000001-wal.1732606033632.temp (wrote 3002 edits, skipped 0 edits in 46 ms) at 1732606037998 (+4 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000000001-wal.1732606033632.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000003002 at 1732606038000 (+2 ms)Processed 3002 edits across 1 Regions in 131 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632, size=0, length=0, corrupted=false, cancelled=false at 1732606038000 2024-11-26T07:27:18,002 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 to hdfs://localhost:41363/hbase/oldWALs/wal.1732606033632 2024-11-26T07:27:18,004 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000003002 2024-11-26T07:27:18,004 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:18,006 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:18,020 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606038006, exclude list is [], retry=0 2024-11-26T07:27:18,023 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:18,024 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:18,024 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:18,026 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606038006 2024-11-26T07:27:18,028 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:18,029 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:18,031 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:18,032 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a7e797a660052076a77697ee9ce9842 columnFamilyName a 2024-11-26T07:27:18,032 DEBUG [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:18,033 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(327): Store=6a7e797a660052076a77697ee9ce9842/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:18,033 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:18,034 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a7e797a660052076a77697ee9ce9842 columnFamilyName b 2024-11-26T07:27:18,034 DEBUG [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:18,035 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(327): Store=6a7e797a660052076a77697ee9ce9842/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:18,035 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:18,037 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a7e797a660052076a77697ee9ce9842 columnFamilyName c 2024-11-26T07:27:18,037 DEBUG [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:18,037 INFO [StoreOpener-6a7e797a660052076a77697ee9ce9842-1 {}] regionserver.HStore(327): Store=6a7e797a660052076a77697ee9ce9842/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:18,038 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:18,039 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:18,040 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:18,041 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000003002 2024-11-26T07:27:18,044 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:18,091 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T07:27:18,477 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a7e797a660052076a77697ee9ce9842 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T07:27:18,516 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/a/01f634ad06dc4f358a3173f30b682086 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732606033664/Put/seqid=0 2024-11-26T07:27:18,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741858_1035 (size=50463) 2024-11-26T07:27:18,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741858_1035 (size=50463) 2024-11-26T07:27:18,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741858_1035 (size=50463) 2024-11-26T07:27:18,531 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/a/01f634ad06dc4f358a3173f30b682086 2024-11-26T07:27:18,539 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/a/01f634ad06dc4f358a3173f30b682086 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/a/01f634ad06dc4f358a3173f30b682086 2024-11-26T07:27:18,546 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/a/01f634ad06dc4f358a3173f30b682086, entries=754, sequenceid=754, filesize=49.3 K 2024-11-26T07:27:18,546 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 6a7e797a660052076a77697ee9ce9842 in 70ms, sequenceid=754, compaction requested=false; wal=null 2024-11-26T07:27:18,561 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T07:27:18,562 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a7e797a660052076a77697ee9ce9842 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T07:27:18,569 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/a/dd7395a015984f8aaadea34512e9f528 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732606033720/Put/seqid=0 2024-11-26T07:27:18,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741859_1036 (size=20072) 2024-11-26T07:27:18,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741859_1036 (size=20072) 2024-11-26T07:27:18,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741859_1036 (size=20072) 2024-11-26T07:27:18,584 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/a/dd7395a015984f8aaadea34512e9f528 2024-11-26T07:27:18,609 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/b/97804b9eb2cd40e6ae04e9adba19c757 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732606033743/Put/seqid=0 2024-11-26T07:27:18,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741860_1037 (size=35835) 2024-11-26T07:27:18,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741860_1037 (size=35835) 2024-11-26T07:27:18,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741860_1037 (size=35835) 2024-11-26T07:27:18,620 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/b/97804b9eb2cd40e6ae04e9adba19c757 2024-11-26T07:27:18,629 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/a/dd7395a015984f8aaadea34512e9f528 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/a/dd7395a015984f8aaadea34512e9f528 2024-11-26T07:27:18,638 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/a/dd7395a015984f8aaadea34512e9f528, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-26T07:27:18,640 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/b/97804b9eb2cd40e6ae04e9adba19c757 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/b/97804b9eb2cd40e6ae04e9adba19c757 2024-11-26T07:27:18,647 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/b/97804b9eb2cd40e6ae04e9adba19c757, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-26T07:27:18,648 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 6a7e797a660052076a77697ee9ce9842 in 86ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-26T07:27:18,658 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T07:27:18,659 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a7e797a660052076a77697ee9ce9842 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T07:27:18,667 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/b/5e895fb4098d425f92cff2afc2571679 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732606033761/Put/seqid=0 2024-11-26T07:27:18,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741861_1038 (size=35082) 2024-11-26T07:27:18,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741861_1038 (size=35082) 2024-11-26T07:27:18,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741861_1038 (size=35082) 2024-11-26T07:27:18,678 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/b/5e895fb4098d425f92cff2afc2571679 2024-11-26T07:27:18,704 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/c/592293fdfafd40d4a50d25e9b7493243 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732606033787/Put/seqid=0 2024-11-26T07:27:18,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741862_1039 (size=20825) 2024-11-26T07:27:18,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741862_1039 (size=20825) 2024-11-26T07:27:18,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741862_1039 (size=20825) 2024-11-26T07:27:18,714 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/c/592293fdfafd40d4a50d25e9b7493243 2024-11-26T07:27:18,722 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/b/5e895fb4098d425f92cff2afc2571679 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/b/5e895fb4098d425f92cff2afc2571679 2024-11-26T07:27:18,729 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/b/5e895fb4098d425f92cff2afc2571679, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-26T07:27:18,730 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/c/592293fdfafd40d4a50d25e9b7493243 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/c/592293fdfafd40d4a50d25e9b7493243 2024-11-26T07:27:18,737 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/c/592293fdfafd40d4a50d25e9b7493243, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-26T07:27:18,737 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 6a7e797a660052076a77697ee9ce9842 in 78ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-26T07:27:18,746 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732606033825/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. 2024-11-26T07:27:18,749 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000003002 2024-11-26T07:27:18,749 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T07:27:18,750 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a7e797a660052076a77697ee9ce9842 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-26T07:27:18,757 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/c/aa375b2d9b8a4e908e71b817ada61040 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732606033795/Put/seqid=0 2024-11-26T07:27:18,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741863_1040 (size=50301) 2024-11-26T07:27:18,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741863_1040 (size=50301) 2024-11-26T07:27:18,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741863_1040 (size=50301) 2024-11-26T07:27:18,766 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/c/aa375b2d9b8a4e908e71b817ada61040 2024-11-26T07:27:18,772 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aa375b2d9b8a4e908e71b817ada61040 2024-11-26T07:27:18,773 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/.tmp/c/aa375b2d9b8a4e908e71b817ada61040 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/c/aa375b2d9b8a4e908e71b817ada61040 2024-11-26T07:27:18,778 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aa375b2d9b8a4e908e71b817ada61040 2024-11-26T07:27:18,778 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/c/aa375b2d9b8a4e908e71b817ada61040, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-26T07:27:18,779 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 6a7e797a660052076a77697ee9ce9842 in 29ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-26T07:27:18,780 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/0000000000000003002 2024-11-26T07:27:18,781 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:18,781 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:18,782 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T07:27:18,784 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a7e797a660052076a77697ee9ce9842 2024-11-26T07:27:18,786 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenIntoWAL/6a7e797a660052076a77697ee9ce9842/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-26T07:27:18,787 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a7e797a660052076a77697ee9ce9842; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73834304, jitterRate=0.10021686553955078}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T07:27:18,787 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a7e797a660052076a77697ee9ce9842: Writing region info on filesystem at 1732606038029Initializing all the Stores at 1732606038030 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606038030Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606038030Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606038030Cleaning up temporary data from old regions at 1732606038781 (+751 ms)Region opened successfully at 1732606038787 (+6 ms) 2024-11-26T07:27:18,840 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a7e797a660052076a77697ee9ce9842, disabling compactions & flushes 2024-11-26T07:27:18,840 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. 2024-11-26T07:27:18,841 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. 2024-11-26T07:27:18,841 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. after waiting 0 ms 2024-11-26T07:27:18,841 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. 2024-11-26T07:27:18,843 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732606033568.6a7e797a660052076a77697ee9ce9842. 2024-11-26T07:27:18,843 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a7e797a660052076a77697ee9ce9842: Waiting for close lock at 1732606038840Disabling compacts and flushes for region at 1732606038840Disabling writes for close at 1732606038841 (+1 ms)Writing region close event to WAL at 1732606038843 (+2 ms)Closed at 1732606038843 2024-11-26T07:27:18,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741857_1034 (size=93) 2024-11-26T07:27:18,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741857_1034 (size=93) 2024-11-26T07:27:18,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741857_1034 (size=93) 2024-11-26T07:27:18,851 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:18,851 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732606038006) 2024-11-26T07:27:18,876 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=402 (was 386) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1193942599_22 at /127.0.0.1:45732 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:45073 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1193942599_22 at /127.0.0.1:42236 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:36583 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36583 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:41363 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1193942599_22 at /127.0.0.1:42192 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1193942599_22 at /127.0.0.1:57766 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45073 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:41363 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=255 (was 242) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8289 (was 8349) 2024-11-26T07:27:18,891 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=402, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=255, ProcessCount=11, AvailableMemoryMB=8287 2024-11-26T07:27:18,914 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:18,917 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:18,918 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:18,920 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57913494, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/hregion-57913494, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:18,934 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57913494/hregion-57913494.1732606038921, exclude list is [], retry=0 2024-11-26T07:27:18,938 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:18,938 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:18,938 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:18,941 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57913494/hregion-57913494.1732606038921 2024-11-26T07:27:18,941 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219)] 2024-11-26T07:27:18,941 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b34d974fbade02c68d69e60fa4327f2b, NAME => 'test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/hbase 2024-11-26T07:27:18,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741865_1042 (size=43) 2024-11-26T07:27:18,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741865_1042 (size=43) 2024-11-26T07:27:18,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741865_1042 (size=43) 2024-11-26T07:27:19,356 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:19,358 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,360 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b34d974fbade02c68d69e60fa4327f2b columnFamilyName a 2024-11-26T07:27:19,360 DEBUG [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:19,360 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(327): Store=b34d974fbade02c68d69e60fa4327f2b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:19,361 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,362 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b34d974fbade02c68d69e60fa4327f2b columnFamilyName b 2024-11-26T07:27:19,362 DEBUG [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:19,363 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(327): Store=b34d974fbade02c68d69e60fa4327f2b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:19,363 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,364 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b34d974fbade02c68d69e60fa4327f2b columnFamilyName c 2024-11-26T07:27:19,364 DEBUG [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:19,365 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(327): Store=b34d974fbade02c68d69e60fa4327f2b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:19,365 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,365 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,366 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,367 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,367 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,367 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:19,369 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,371 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:19,371 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b34d974fbade02c68d69e60fa4327f2b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70435288, jitterRate=0.049567580223083496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:19,373 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b34d974fbade02c68d69e60fa4327f2b: Writing region info on filesystem at 1732606039357Initializing all the Stores at 1732606039358 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606039358Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606039358Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606039358Cleaning up temporary data from old regions at 1732606039367 (+9 ms)Region opened successfully at 1732606039373 (+6 ms) 2024-11-26T07:27:19,373 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b34d974fbade02c68d69e60fa4327f2b, disabling compactions & flushes 2024-11-26T07:27:19,373 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. 2024-11-26T07:27:19,373 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. 2024-11-26T07:27:19,373 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. after waiting 0 ms 2024-11-26T07:27:19,373 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. 2024-11-26T07:27:19,374 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. 2024-11-26T07:27:19,374 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b34d974fbade02c68d69e60fa4327f2b: Waiting for close lock at 1732606039373Disabling compacts and flushes for region at 1732606039373Disabling writes for close at 1732606039373Writing region close event to WAL at 1732606039374 (+1 ms)Closed at 1732606039374 2024-11-26T07:27:19,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741864_1041 (size=93) 2024-11-26T07:27:19,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741864_1041 (size=93) 2024-11-26T07:27:19,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741864_1041 (size=93) 2024-11-26T07:27:19,379 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:19,379 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57913494:(num 1732606038921) 2024-11-26T07:27:19,379 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:19,381 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:19,395 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381, exclude list is [], retry=0 2024-11-26T07:27:19,398 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:19,398 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:19,399 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:19,401 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381 2024-11-26T07:27:19,401 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:19,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741866_1043 (size=200357) 2024-11-26T07:27:19,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741866_1043 (size=200357) 2024-11-26T07:27:19,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741866_1043 (size=200357) 2024-11-26T07:27:19,553 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381, size=195.7 K (200357bytes) 2024-11-26T07:27:19,553 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381 2024-11-26T07:27:19,553 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381 after 0ms 2024-11-26T07:27:19,556 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:19,558 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381 took 5ms 2024-11-26T07:27:19,562 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732606039381.temp 2024-11-26T07:27:19,565 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000000001-wal.1732606039381.temp 2024-11-26T07:27:19,612 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381 so closing down 2024-11-26T07:27:19,612 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:19,612 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:19,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741867_1044 (size=200357) 2024-11-26T07:27:19,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741867_1044 (size=200357) 2024-11-26T07:27:19,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741867_1044 (size=200357) 2024-11-26T07:27:19,618 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000000001-wal.1732606039381.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-11-26T07:27:19,620 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000000001-wal.1732606039381.temp to hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003000 2024-11-26T07:27:19,621 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-11-26T07:27:19,621 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381, journal: Splitting hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381, size=195.7 K (200357bytes) at 1732606039553Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000000001-wal.1732606039381.temp at 1732606039565 (+12 ms)Split 1024 edits, skipped 0 edits. at 1732606039577 (+12 ms)Split 2048 edits, skipped 0 edits. at 1732606039596 (+19 ms)Finishing writing output for hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381 so closing down at 1732606039612 (+16 ms)3 split writer threads finished at 1732606039612Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000000001-wal.1732606039381.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1732606039618 (+6 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000000001-wal.1732606039381.temp to hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003000 at 1732606039620 (+2 ms)Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1732606039621 (+1 ms) 2024-11-26T07:27:19,623 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039381 to hdfs://localhost:41363/hbase/oldWALs/wal.1732606039381 2024-11-26T07:27:19,624 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003000 2024-11-26T07:27:19,624 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:19,627 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:19,647 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627, exclude list is [], retry=0 2024-11-26T07:27:19,651 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:19,652 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:19,652 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:19,654 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627 2024-11-26T07:27:19,654 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:19,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741868_1045 (size=200484) 2024-11-26T07:27:19,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741868_1045 (size=200484) 2024-11-26T07:27:19,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741868_1045 (size=200484) 2024-11-26T07:27:19,810 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627, size=195.8 K (200484bytes) 2024-11-26T07:27:19,810 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627 2024-11-26T07:27:19,811 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627 after 0ms 2024-11-26T07:27:19,814 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:19,816 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627 took 6ms 2024-11-26T07:27:19,824 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732606039627.temp 2024-11-26T07:27:19,830 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003001-wal.1732606039627.temp 2024-11-26T07:27:19,877 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627 so closing down 2024-11-26T07:27:19,877 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:19,878 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:19,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741869_1046 (size=200484) 2024-11-26T07:27:19,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741869_1046 (size=200484) 2024-11-26T07:27:19,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741869_1046 (size=200484) 2024-11-26T07:27:19,887 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003001-wal.1732606039627.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-11-26T07:27:19,889 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003001-wal.1732606039627.temp to hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000006000 2024-11-26T07:27:19,890 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-11-26T07:27:19,890 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627, journal: Splitting hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627, size=195.8 K (200484bytes) at 1732606039810Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003001-wal.1732606039627.temp at 1732606039830 (+20 ms)Split 1024 edits, skipped 0 edits. at 1732606039841 (+11 ms)Split 2048 edits, skipped 0 edits. at 1732606039859 (+18 ms)Finishing writing output for hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627 so closing down at 1732606039877 (+18 ms)3 split writer threads finished at 1732606039878 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003001-wal.1732606039627.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1732606039887 (+9 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003001-wal.1732606039627.temp to hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000006000 at 1732606039889 (+2 ms)Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1732606039890 (+1 ms) 2024-11-26T07:27:19,892 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039627 to hdfs://localhost:41363/hbase/oldWALs/wal.1732606039627 2024-11-26T07:27:19,893 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000006000 2024-11-26T07:27:19,894 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:19,896 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/test2727-manual,16010,1732606038913, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:19,919 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039897, exclude list is [], retry=0 2024-11-26T07:27:19,923 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:19,923 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:19,924 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:19,926 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732606038913/wal.1732606039897 2024-11-26T07:27:19,927 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:19,927 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b34d974fbade02c68d69e60fa4327f2b, NAME => 'test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:19,927 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:19,927 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,927 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,929 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,930 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b34d974fbade02c68d69e60fa4327f2b columnFamilyName a 2024-11-26T07:27:19,930 DEBUG [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:19,931 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(327): Store=b34d974fbade02c68d69e60fa4327f2b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:19,931 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,932 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b34d974fbade02c68d69e60fa4327f2b columnFamilyName b 2024-11-26T07:27:19,932 DEBUG [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:19,933 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(327): Store=b34d974fbade02c68d69e60fa4327f2b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:19,933 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,934 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b34d974fbade02c68d69e60fa4327f2b columnFamilyName c 2024-11-26T07:27:19,934 DEBUG [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:19,935 INFO [StoreOpener-b34d974fbade02c68d69e60fa4327f2b-1 {}] regionserver.HStore(327): Store=b34d974fbade02c68d69e60fa4327f2b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:19,935 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,936 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,938 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:19,939 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003000 2024-11-26T07:27:19,942 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:19,990 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003000 2024-11-26T07:27:19,991 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000006000 2024-11-26T07:27:19,994 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:20,042 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000006000 2024-11-26T07:27:20,042 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b34d974fbade02c68d69e60fa4327f2b 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-26T07:27:20,068 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/.tmp/a/b900d65d50fd4922b07a639b2ab3689c is 41, key is test2727/a:100/1732606039658/Put/seqid=0 2024-11-26T07:27:20,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741871_1048 (size=84227) 2024-11-26T07:27:20,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741871_1048 (size=84227) 2024-11-26T07:27:20,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741871_1048 (size=84227) 2024-11-26T07:27:20,081 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/.tmp/a/b900d65d50fd4922b07a639b2ab3689c 2024-11-26T07:27:20,123 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/.tmp/b/87a28ae98d104c3896b6f297c81c2389 is 41, key is test2727/b:100/1732606039699/Put/seqid=0 2024-11-26T07:27:20,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741872_1049 (size=84609) 2024-11-26T07:27:20,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741872_1049 (size=84609) 2024-11-26T07:27:20,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741872_1049 (size=84609) 2024-11-26T07:27:20,131 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/.tmp/b/87a28ae98d104c3896b6f297c81c2389 2024-11-26T07:27:20,159 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/.tmp/c/b5be16a6bf8740d3b126e95474e13649 is 41, key is test2727/c:100/1732606039742/Put/seqid=0 2024-11-26T07:27:20,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741873_1050 (size=84609) 2024-11-26T07:27:20,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741873_1050 (size=84609) 2024-11-26T07:27:20,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741873_1050 (size=84609) 2024-11-26T07:27:20,168 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/.tmp/c/b5be16a6bf8740d3b126e95474e13649 2024-11-26T07:27:20,181 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/.tmp/a/b900d65d50fd4922b07a639b2ab3689c as hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/a/b900d65d50fd4922b07a639b2ab3689c 2024-11-26T07:27:20,187 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T07:27:20,187 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T07:27:20,188 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/a/b900d65d50fd4922b07a639b2ab3689c, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-26T07:27:20,190 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T07:27:20,190 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T07:27:20,190 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T07:27:20,190 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T07:27:20,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-26T07:27:20,192 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-26T07:27:20,194 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/.tmp/b/87a28ae98d104c3896b6f297c81c2389 as hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/b/87a28ae98d104c3896b6f297c81c2389 2024-11-26T07:27:20,204 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/b/87a28ae98d104c3896b6f297c81c2389, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-26T07:27:20,206 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/.tmp/c/b5be16a6bf8740d3b126e95474e13649 as hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/c/b5be16a6bf8740d3b126e95474e13649 2024-11-26T07:27:20,217 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/c/b5be16a6bf8740d3b126e95474e13649, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-26T07:27:20,218 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for b34d974fbade02c68d69e60fa4327f2b in 176ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-26T07:27:20,219 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000003000 2024-11-26T07:27:20,220 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/0000000000000006000 2024-11-26T07:27:20,222 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:20,222 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:20,223 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:20,233 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b34d974fbade02c68d69e60fa4327f2b 2024-11-26T07:27:20,242 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/test2727/b34d974fbade02c68d69e60fa4327f2b/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-26T07:27:20,244 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b34d974fbade02c68d69e60fa4327f2b; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60529128, jitterRate=-0.0980457067489624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:20,246 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b34d974fbade02c68d69e60fa4327f2b: Writing region info on filesystem at 1732606039927Initializing all the Stores at 1732606039928 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606039929 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606039929Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606039929Obtaining lock to block concurrent updates at 1732606040042 (+113 ms)Preparing flush snapshotting stores in b34d974fbade02c68d69e60fa4327f2b at 1732606040042Finished memstore snapshotting test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732606040042Flushing stores of test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. at 1732606040042Flushing b34d974fbade02c68d69e60fa4327f2b/a: creating writer at 1732606040042Flushing b34d974fbade02c68d69e60fa4327f2b/a: appending metadata at 1732606040067 (+25 ms)Flushing b34d974fbade02c68d69e60fa4327f2b/a: closing flushed file at 1732606040068 (+1 ms)Flushing b34d974fbade02c68d69e60fa4327f2b/b: creating writer at 1732606040088 (+20 ms)Flushing b34d974fbade02c68d69e60fa4327f2b/b: appending metadata at 1732606040120 (+32 ms)Flushing b34d974fbade02c68d69e60fa4327f2b/b: closing flushed file at 1732606040120Flushing b34d974fbade02c68d69e60fa4327f2b/c: creating writer at 1732606040138 (+18 ms)Flushing b34d974fbade02c68d69e60fa4327f2b/c: appending metadata at 1732606040158 (+20 ms)Flushing b34d974fbade02c68d69e60fa4327f2b/c: closing flushed file at 1732606040158Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c9de81b: reopening flushed file at 1732606040180 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cca98c9: reopening flushed file at 1732606040188 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d8bf9a5: reopening flushed file at 1732606040204 (+16 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for b34d974fbade02c68d69e60fa4327f2b in 176ms, sequenceid=6000, compaction requested=false; wal=null at 1732606040218 (+14 ms)Cleaning up temporary data from old regions at 1732606040222 (+4 ms)Region opened successfully at 1732606040246 (+24 ms) 2024-11-26T07:27:20,247 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-26T07:27:20,248 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b34d974fbade02c68d69e60fa4327f2b, disabling compactions & flushes 2024-11-26T07:27:20,248 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. 2024-11-26T07:27:20,248 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. 2024-11-26T07:27:20,248 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. after waiting 0 ms 2024-11-26T07:27:20,248 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. 2024-11-26T07:27:20,250 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732606038915.b34d974fbade02c68d69e60fa4327f2b. 2024-11-26T07:27:20,251 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b34d974fbade02c68d69e60fa4327f2b: Waiting for close lock at 1732606040247Disabling compacts and flushes for region at 1732606040247Disabling writes for close at 1732606040248 (+1 ms)Writing region close event to WAL at 1732606040250 (+2 ms)Closed at 1732606040250 2024-11-26T07:27:20,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741870_1047 (size=93) 2024-11-26T07:27:20,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741870_1047 (size=93) 2024-11-26T07:27:20,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741870_1047 (size=93) 2024-11-26T07:27:20,260 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:20,260 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732606039897) 2024-11-26T07:27:20,283 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=404 (was 402) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:45828 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:42346 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:57766 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=255 (was 255), ProcessCount=11 (was 11), AvailableMemoryMB=8132 (was 8287) 2024-11-26T07:27:20,298 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=255, ProcessCount=11, AvailableMemoryMB=8131 2024-11-26T07:27:20,324 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:20,332 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:20,333 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732606040333 2024-11-26T07:27:20,346 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333 2024-11-26T07:27:20,352 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:46219:46219)] 2024-11-26T07:27:20,357 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 85dab9a695634aedca83aaf2c00e05ed, NAME => 'testSequentialEditLogSeqNum,,1732606040325.85dab9a695634aedca83aaf2c00e05ed.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:20,358 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732606040325.85dab9a695634aedca83aaf2c00e05ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:20,358 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 85dab9a695634aedca83aaf2c00e05ed 2024-11-26T07:27:20,358 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 85dab9a695634aedca83aaf2c00e05ed 2024-11-26T07:27:20,359 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed doesn't exist for region: 85dab9a695634aedca83aaf2c00e05ed on table testSequentialEditLogSeqNum 2024-11-26T07:27:20,360 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 85dab9a695634aedca83aaf2c00e05ed on table testSequentialEditLogSeqNum 2024-11-26T07:27:20,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741875_1052 (size=62) 2024-11-26T07:27:20,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741875_1052 (size=62) 2024-11-26T07:27:20,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741875_1052 (size=62) 2024-11-26T07:27:20,374 INFO [StoreOpener-85dab9a695634aedca83aaf2c00e05ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 85dab9a695634aedca83aaf2c00e05ed 2024-11-26T07:27:20,376 INFO [StoreOpener-85dab9a695634aedca83aaf2c00e05ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85dab9a695634aedca83aaf2c00e05ed columnFamilyName a 2024-11-26T07:27:20,376 DEBUG [StoreOpener-85dab9a695634aedca83aaf2c00e05ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:20,376 INFO [StoreOpener-85dab9a695634aedca83aaf2c00e05ed-1 {}] regionserver.HStore(327): Store=85dab9a695634aedca83aaf2c00e05ed/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:20,377 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 85dab9a695634aedca83aaf2c00e05ed 2024-11-26T07:27:20,377 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed 2024-11-26T07:27:20,378 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed 2024-11-26T07:27:20,378 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 85dab9a695634aedca83aaf2c00e05ed 2024-11-26T07:27:20,378 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 85dab9a695634aedca83aaf2c00e05ed 2024-11-26T07:27:20,381 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 85dab9a695634aedca83aaf2c00e05ed 2024-11-26T07:27:20,383 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:20,384 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 85dab9a695634aedca83aaf2c00e05ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74084381, jitterRate=0.10394330322742462}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:27:20,386 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 85dab9a695634aedca83aaf2c00e05ed: Writing region info on filesystem at 1732606040358Initializing all the Stores at 1732606040373 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606040373Cleaning up temporary data from old regions at 1732606040378 (+5 ms)Region opened successfully at 1732606040386 (+8 ms) 2024-11-26T07:27:20,401 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 85dab9a695634aedca83aaf2c00e05ed 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-26T07:27:20,428 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/.tmp/a/e6e01e37815c41ae9944e1032906886a is 81, key is testSequentialEditLogSeqNum/a:x0/1732606040386/Put/seqid=0 2024-11-26T07:27:20,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741876_1053 (size=5833) 2024-11-26T07:27:20,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741876_1053 (size=5833) 2024-11-26T07:27:20,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741876_1053 (size=5833) 2024-11-26T07:27:20,437 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/.tmp/a/e6e01e37815c41ae9944e1032906886a 2024-11-26T07:27:20,445 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/.tmp/a/e6e01e37815c41ae9944e1032906886a as hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/a/e6e01e37815c41ae9944e1032906886a 2024-11-26T07:27:20,452 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/a/e6e01e37815c41ae9944e1032906886a, entries=10, sequenceid=13, filesize=5.7 K 2024-11-26T07:27:20,454 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 85dab9a695634aedca83aaf2c00e05ed in 53ms, sequenceid=13, compaction requested=false 2024-11-26T07:27:20,454 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 85dab9a695634aedca83aaf2c00e05ed: 2024-11-26T07:27:20,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T07:27:20,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T07:27:20,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T07:27:20,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T07:27:20,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T07:27:20,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741874_1051 (size=1616) 2024-11-26T07:27:20,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741874_1051 (size=1616) 2024-11-26T07:27:20,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741874_1051 (size=1616) 2024-11-26T07:27:20,486 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333, size=1.6 K (1616bytes) 2024-11-26T07:27:20,486 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333 2024-11-26T07:27:20,486 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333 after 0ms 2024-11-26T07:27:20,489 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:20,490 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333 took 5ms 2024-11-26T07:27:20,492 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333 so closing down 2024-11-26T07:27:20,492 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:20,493 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732606040333.temp 2024-11-26T07:27:20,495 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/recovered.edits/0000000000000000003-wal.1732606040333.temp 2024-11-26T07:27:20,496 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:20,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741877_1054 (size=1175) 2024-11-26T07:27:20,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741877_1054 (size=1175) 2024-11-26T07:27:20,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741877_1054 (size=1175) 2024-11-26T07:27:20,514 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/recovered.edits/0000000000000000003-wal.1732606040333.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:20,516 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/recovered.edits/0000000000000000003-wal.1732606040333.temp to hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/recovered.edits/0000000000000000020 2024-11-26T07:27:20,516 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 26 ms; skipped=2; WAL=hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-11-26T07:27:20,516 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333, journal: Splitting hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333, size=1.6 K (1616bytes) at 1732606040486Finishing writing output for hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333 so closing down at 1732606040492 (+6 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/recovered.edits/0000000000000000003-wal.1732606040333.temp at 1732606040495 (+3 ms)3 split writer threads finished at 1732606040496 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/recovered.edits/0000000000000000003-wal.1732606040333.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732606040514 (+18 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/recovered.edits/0000000000000000003-wal.1732606040333.temp to hdfs://localhost:41363/hbase/data/default/testSequentialEditLogSeqNum/85dab9a695634aedca83aaf2c00e05ed/recovered.edits/0000000000000000020 at 1732606040516 (+2 ms)Processed 17 edits across 1 Regions in 26 ms; skipped=2; WAL=hdfs://localhost:41363/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732606040323/wal.1732606040333, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1732606040516 2024-11-26T07:27:20,537 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:45828 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:42346 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:57766 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=255 (was 255), ProcessCount=11 (was 11), AvailableMemoryMB=8108 (was 8131) 2024-11-26T07:27:20,551 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=255, ProcessCount=11, AvailableMemoryMB=8107 2024-11-26T07:27:20,567 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:20,570 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:20,593 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:20,596 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33753899, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/hregion-33753899, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:20,613 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33753899/hregion-33753899.1732606040597, exclude list is [], retry=0 2024-11-26T07:27:20,616 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:20,616 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:20,618 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:20,621 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33753899/hregion-33753899.1732606040597 2024-11-26T07:27:20,624 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:20,625 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => eb979a5436d660374ab1ff5cac20e35a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/hbase 2024-11-26T07:27:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741879_1056 (size=70) 2024-11-26T07:27:20,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741879_1056 (size=70) 2024-11-26T07:27:20,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741879_1056 (size=70) 2024-11-26T07:27:20,642 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:20,644 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,646 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb979a5436d660374ab1ff5cac20e35a columnFamilyName a 2024-11-26T07:27:20,646 DEBUG [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:20,647 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(327): Store=eb979a5436d660374ab1ff5cac20e35a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:20,647 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,648 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb979a5436d660374ab1ff5cac20e35a columnFamilyName b 2024-11-26T07:27:20,648 DEBUG [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:20,649 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(327): Store=eb979a5436d660374ab1ff5cac20e35a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:20,649 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,651 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb979a5436d660374ab1ff5cac20e35a columnFamilyName c 2024-11-26T07:27:20,651 DEBUG [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:20,651 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(327): Store=eb979a5436d660374ab1ff5cac20e35a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:20,651 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,652 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,653 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,654 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,654 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,654 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:20,655 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,658 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:20,658 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eb979a5436d660374ab1ff5cac20e35a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69034854, jitterRate=0.028699487447738647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:20,660 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eb979a5436d660374ab1ff5cac20e35a: Writing region info on filesystem at 1732606040642Initializing all the Stores at 1732606040643 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606040643Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606040644 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606040644Cleaning up temporary data from old regions at 1732606040654 (+10 ms)Region opened successfully at 1732606040659 (+5 ms) 2024-11-26T07:27:20,660 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing eb979a5436d660374ab1ff5cac20e35a, disabling compactions & flushes 2024-11-26T07:27:20,660 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. 2024-11-26T07:27:20,660 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. 2024-11-26T07:27:20,660 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. after waiting 0 ms 2024-11-26T07:27:20,660 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. 2024-11-26T07:27:20,660 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. 2024-11-26T07:27:20,660 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for eb979a5436d660374ab1ff5cac20e35a: Waiting for close lock at 1732606040660Disabling compacts and flushes for region at 1732606040660Disabling writes for close at 1732606040660Writing region close event to WAL at 1732606040660Closed at 1732606040660 2024-11-26T07:27:20,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741878_1055 (size=93) 2024-11-26T07:27:20,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741878_1055 (size=93) 2024-11-26T07:27:20,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741878_1055 (size=93) 2024-11-26T07:27:20,666 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:20,666 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33753899:(num 1732606040597) 2024-11-26T07:27:20,666 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:20,669 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:20,684 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669, exclude list is [], retry=0 2024-11-26T07:27:20,687 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:20,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:20,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:20,690 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 2024-11-26T07:27:20,690 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:20,690 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => eb979a5436d660374ab1ff5cac20e35a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:20,691 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:20,691 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,691 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,692 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,693 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb979a5436d660374ab1ff5cac20e35a columnFamilyName a 2024-11-26T07:27:20,693 DEBUG [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:20,694 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(327): Store=eb979a5436d660374ab1ff5cac20e35a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:20,694 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,695 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb979a5436d660374ab1ff5cac20e35a columnFamilyName b 2024-11-26T07:27:20,695 DEBUG [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:20,695 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(327): Store=eb979a5436d660374ab1ff5cac20e35a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:20,696 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,696 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb979a5436d660374ab1ff5cac20e35a columnFamilyName c 2024-11-26T07:27:20,696 DEBUG [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:20,697 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(327): Store=eb979a5436d660374ab1ff5cac20e35a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:20,697 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,698 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,699 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,700 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,700 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,701 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:20,703 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:20,704 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eb979a5436d660374ab1ff5cac20e35a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67749350, jitterRate=0.00954398512840271}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:20,705 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eb979a5436d660374ab1ff5cac20e35a: Writing region info on filesystem at 1732606040691Initializing all the Stores at 1732606040692 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606040692Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606040692Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606040692Cleaning up temporary data from old regions at 1732606040700 (+8 ms)Region opened successfully at 1732606040705 (+5 ms) 2024-11-26T07:27:20,712 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732606040710/Put/seqid=0 2024-11-26T07:27:20,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741881_1058 (size=4826) 2024-11-26T07:27:20,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741881_1058 (size=4826) 2024-11-26T07:27:20,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741881_1058 (size=4826) 2024-11-26T07:27:20,723 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41363/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in eb979a5436d660374ab1ff5cac20e35a/a 2024-11-26T07:27:20,733 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-26T07:27:20,734 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T07:27:20,734 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for eb979a5436d660374ab1ff5cac20e35a: 2024-11-26T07:27:20,736 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/a/8ee685e3b6964f79a080fbdb8992f607_SeqId_3_ 2024-11-26T07:27:20,737 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41363/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into eb979a5436d660374ab1ff5cac20e35a/a as hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/a/8ee685e3b6964f79a080fbdb8992f607_SeqId_3_ - updating store file list. 2024-11-26T07:27:20,742 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8ee685e3b6964f79a080fbdb8992f607_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-26T07:27:20,743 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/a/8ee685e3b6964f79a080fbdb8992f607_SeqId_3_ into eb979a5436d660374ab1ff5cac20e35a/a 2024-11-26T07:27:20,743 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41363/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into eb979a5436d660374ab1ff5cac20e35a/a (new location: hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/a/8ee685e3b6964f79a080fbdb8992f607_SeqId_3_) 2024-11-26T07:27:20,790 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669, size=0 (0bytes) 2024-11-26T07:27:20,790 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 might be still open, length is 0 2024-11-26T07:27:20,790 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 2024-11-26T07:27:20,790 WARN [IPC Server handler 2 on default port 41363 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-26T07:27:20,791 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 after 1ms 2024-11-26T07:27:23,703 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:54486 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:42007:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54486 dst: /127.0.0.1:42007 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42007 remote=/127.0.0.1:54486]. Total timeout mills is 60000, 57046 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:23,704 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:59178 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40873:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59178 dst: /127.0.0.1:40873 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:23,704 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:59094 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59094 dst: /127.0.0.1:41787 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741880_1059 (size=434) 2024-11-26T07:27:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741880_1059 (size=434) 2024-11-26T07:27:24,791 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 after 4001ms 2024-11-26T07:27:24,795 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:24,795 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 took 4006ms 2024-11-26T07:27:24,797 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669; continuing. 2024-11-26T07:27:24,797 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 so closing down 2024-11-26T07:27:24,798 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:24,800 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732606040669.temp 2024-11-26T07:27:24,801 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005-wal.1732606040669.temp 2024-11-26T07:27:24,801 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:24,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741882_1060 (size=236) 2024-11-26T07:27:24,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741882_1060 (size=236) 2024-11-26T07:27:24,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741882_1060 (size=236) 2024-11-26T07:27:24,809 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005-wal.1732606040669.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:24,811 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005-wal.1732606040669.temp to hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005 2024-11-26T07:27:24,811 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T07:27:24,811 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669, journal: Splitting hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669, size=0 (0bytes) at 1732606040790Finishing writing output for hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 so closing down at 1732606044797 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005-wal.1732606040669.temp at 1732606044801 (+4 ms)3 split writer threads finished at 1732606044801Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005-wal.1732606040669.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732606044809 (+8 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005-wal.1732606040669.temp to hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005 at 1732606044811 (+2 ms)Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669, size=0, length=0, corrupted=false, cancelled=false at 1732606044811 2024-11-26T07:27:24,813 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 to hdfs://localhost:41363/hbase/oldWALs/wal.1732606040669 2024-11-26T07:27:24,814 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005 2024-11-26T07:27:24,814 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:24,815 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:24,830 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606044816, exclude list is [], retry=0 2024-11-26T07:27:24,833 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:24,833 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:24,834 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:24,836 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606044816 2024-11-26T07:27:24,836 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:24,836 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => eb979a5436d660374ab1ff5cac20e35a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:24,836 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:24,837 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,837 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,838 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,839 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb979a5436d660374ab1ff5cac20e35a columnFamilyName a 2024-11-26T07:27:24,839 DEBUG [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:24,846 DEBUG [StoreFileOpener-eb979a5436d660374ab1ff5cac20e35a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8ee685e3b6964f79a080fbdb8992f607_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-26T07:27:24,846 DEBUG [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/a/8ee685e3b6964f79a080fbdb8992f607_SeqId_3_ 2024-11-26T07:27:24,846 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(327): Store=eb979a5436d660374ab1ff5cac20e35a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:24,846 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,847 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb979a5436d660374ab1ff5cac20e35a columnFamilyName b 2024-11-26T07:27:24,847 DEBUG [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:24,848 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(327): Store=eb979a5436d660374ab1ff5cac20e35a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:24,848 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,848 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb979a5436d660374ab1ff5cac20e35a columnFamilyName c 2024-11-26T07:27:24,849 DEBUG [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:24,849 INFO [StoreOpener-eb979a5436d660374ab1ff5cac20e35a-1 {}] regionserver.HStore(327): Store=eb979a5436d660374ab1ff5cac20e35a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:24,849 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,850 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,851 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,852 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005 2024-11-26T07:27:24,854 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:24,856 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005 2024-11-26T07:27:24,856 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eb979a5436d660374ab1ff5cac20e35a 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-26T07:27:24,872 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/.tmp/a/f9f83a900fbb4652a9bc71978d0029c4 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732606040748/Put/seqid=0 2024-11-26T07:27:24,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741884_1062 (size=5149) 2024-11-26T07:27:24,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741884_1062 (size=5149) 2024-11-26T07:27:24,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741884_1062 (size=5149) 2024-11-26T07:27:24,906 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/.tmp/a/f9f83a900fbb4652a9bc71978d0029c4 2024-11-26T07:27:24,912 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/.tmp/a/f9f83a900fbb4652a9bc71978d0029c4 as hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/a/f9f83a900fbb4652a9bc71978d0029c4 2024-11-26T07:27:24,917 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/a/f9f83a900fbb4652a9bc71978d0029c4, entries=1, sequenceid=5, filesize=5.0 K 2024-11-26T07:27:24,918 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for eb979a5436d660374ab1ff5cac20e35a in 61ms, sequenceid=5, compaction requested=false; wal=null 2024-11-26T07:27:24,918 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/0000000000000000005 2024-11-26T07:27:24,919 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,919 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,920 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:24,921 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eb979a5436d660374ab1ff5cac20e35a 2024-11-26T07:27:24,924 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/eb979a5436d660374ab1ff5cac20e35a/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-26T07:27:24,924 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eb979a5436d660374ab1ff5cac20e35a; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71155001, jitterRate=0.060292139649391174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:24,925 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eb979a5436d660374ab1ff5cac20e35a: Writing region info on filesystem at 1732606044837Initializing all the Stores at 1732606044838 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606044838Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606044838Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606044838Obtaining lock to block concurrent updates at 1732606044856 (+18 ms)Preparing flush snapshotting stores in eb979a5436d660374ab1ff5cac20e35a at 1732606044856Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732606044856Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. at 1732606044856Flushing eb979a5436d660374ab1ff5cac20e35a/a: creating writer at 1732606044856Flushing eb979a5436d660374ab1ff5cac20e35a/a: appending metadata at 1732606044871 (+15 ms)Flushing eb979a5436d660374ab1ff5cac20e35a/a: closing flushed file at 1732606044871Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ccaa43e: reopening flushed file at 1732606044911 (+40 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for eb979a5436d660374ab1ff5cac20e35a in 61ms, sequenceid=5, compaction requested=false; wal=null at 1732606044918 (+7 ms)Cleaning up temporary data from old regions at 1732606044919 (+1 ms)Region opened successfully at 1732606044925 (+6 ms) 2024-11-26T07:27:24,928 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing eb979a5436d660374ab1ff5cac20e35a, disabling compactions & flushes 2024-11-26T07:27:24,929 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. 2024-11-26T07:27:24,929 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. 2024-11-26T07:27:24,929 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. after waiting 0 ms 2024-11-26T07:27:24,929 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. 2024-11-26T07:27:24,929 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732606040568.eb979a5436d660374ab1ff5cac20e35a. 2024-11-26T07:27:24,929 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for eb979a5436d660374ab1ff5cac20e35a: Waiting for close lock at 1732606044928Disabling compacts and flushes for region at 1732606044928Disabling writes for close at 1732606044929 (+1 ms)Writing region close event to WAL at 1732606044929Closed at 1732606044929 2024-11-26T07:27:24,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741883_1061 (size=93) 2024-11-26T07:27:24,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741883_1061 (size=93) 2024-11-26T07:27:24,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741883_1061 (size=93) 2024-11-26T07:27:24,934 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:24,935 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732606044816) 2024-11-26T07:27:24,950 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 409) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-539939899_22 at /127.0.0.1:59130 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-539939899_22 at /127.0.0.1:54526 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:41363 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:41363 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-539939899_22 at /127.0.0.1:59196 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 255), ProcessCount=11 (was 11), AvailableMemoryMB=8003 (was 8107) 2024-11-26T07:27:24,961 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=8002 2024-11-26T07:27:24,977 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:24,982 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T07:27:24,986 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 15ed1af4400d,45945,1732606029786 2024-11-26T07:27:24,989 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6600706f 2024-11-26T07:27:24,990 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T07:27:24,992 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T07:27:24,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T07:27:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-26T07:27:25,007 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T07:27:25,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-26T07:27:25,010 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:25,011 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T07:27:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:27:25,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741885_1063 (size=694) 2024-11-26T07:27:25,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741885_1063 (size=694) 2024-11-26T07:27:25,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741885_1063 (size=694) 2024-11-26T07:27:25,025 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 28da289971d201b5d537d47956344363, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42 2024-11-26T07:27:25,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741886_1064 (size=77) 2024-11-26T07:27:25,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741886_1064 (size=77) 2024-11-26T07:27:25,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741886_1064 (size=77) 2024-11-26T07:27:25,034 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:25,034 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 28da289971d201b5d537d47956344363, disabling compactions & flushes 2024-11-26T07:27:25,034 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,034 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,034 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. after waiting 0 ms 2024-11-26T07:27:25,034 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,034 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,034 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 28da289971d201b5d537d47956344363: Waiting for close lock at 1732606045034Disabling compacts and flushes for region at 1732606045034Disabling writes for close at 1732606045034Writing region close event to WAL at 1732606045034Closed at 1732606045034 2024-11-26T07:27:25,036 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T07:27:25,041 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732606045036"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732606045036"}]},"ts":"1732606045036"} 2024-11-26T07:27:25,045 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T07:27:25,046 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T07:27:25,049 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732606045046"}]},"ts":"1732606045046"} 2024-11-26T07:27:25,054 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-26T07:27:25,054 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {15ed1af4400d=0} racks are {/default-rack=0} 2024-11-26T07:27:25,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T07:27:25,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T07:27:25,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T07:27:25,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T07:27:25,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T07:27:25,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T07:27:25,056 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T07:27:25,056 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T07:27:25,056 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T07:27:25,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T07:27:25,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, ASSIGN}] 2024-11-26T07:27:25,059 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, ASSIGN 2024-11-26T07:27:25,060 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, ASSIGN; state=OFFLINE, location=15ed1af4400d,42563,1732606030451; forceNewPlan=false, retain=false 2024-11-26T07:27:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:27:25,214 INFO [15ed1af4400d:45945 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T07:27:25,215 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=OPENING, regionLocation=15ed1af4400d,42563,1732606030451 2024-11-26T07:27:25,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, ASSIGN because future has completed 2024-11-26T07:27:25,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,42563,1732606030451}] 2024-11-26T07:27:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:27:25,372 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T07:27:25,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54619, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T07:27:25,379 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,379 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 28da289971d201b5d537d47956344363, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:25,380 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,380 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:25,380 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,380 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,382 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,384 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf1 2024-11-26T07:27:25,384 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:25,384 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:25,384 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,386 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf2 2024-11-26T07:27:25,386 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:25,386 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:25,386 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,387 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363 2024-11-26T07:27:25,387 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363 2024-11-26T07:27:25,388 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,388 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,389 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T07:27:25,390 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,392 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:25,393 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 28da289971d201b5d537d47956344363; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72577140, jitterRate=0.08148366212844849}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T07:27:25,393 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,394 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 28da289971d201b5d537d47956344363: Running coprocessor pre-open hook at 1732606045381Writing region info on filesystem at 1732606045381Initializing all the Stores at 1732606045382 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606045382Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606045382Cleaning up temporary data from old regions at 1732606045388 (+6 ms)Running coprocessor post-open hooks at 1732606045393 (+5 ms)Region opened successfully at 1732606045394 (+1 ms) 2024-11-26T07:27:25,396 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., pid=6, masterSystemTime=1732606045372 2024-11-26T07:27:25,398 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,399 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,399 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=OPEN, openSeqNum=2, regionLocation=15ed1af4400d,42563,1732606030451 2024-11-26T07:27:25,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,42563,1732606030451 because future has completed 2024-11-26T07:27:25,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T07:27:25,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,42563,1732606030451 in 185 msec 2024-11-26T07:27:25,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T07:27:25,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, ASSIGN in 349 msec 2024-11-26T07:27:25,411 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T07:27:25,411 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732606045411"}]},"ts":"1732606045411"} 2024-11-26T07:27:25,414 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-26T07:27:25,415 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T07:27:25,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 416 msec 2024-11-26T07:27:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:27:25,641 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-26T07:27:25,641 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-26T07:27:25,643 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T07:27:25,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-26T07:27:25,649 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T07:27:25,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-26T07:27:25,662 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=2] 2024-11-26T07:27:25,663 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T07:27:25,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T07:27:25,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=28da289971d201b5d537d47956344363, source=15ed1af4400d,42563,1732606030451, destination=15ed1af4400d,38495,1732606030549, warming up region on 15ed1af4400d,38495,1732606030549 2024-11-26T07:27:25,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=28da289971d201b5d537d47956344363, source=15ed1af4400d,42563,1732606030451, destination=15ed1af4400d,38495,1732606030549, running balancer 2024-11-26T07:27:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE 2024-11-26T07:27:25,684 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE 2024-11-26T07:27:25,686 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=CLOSING, regionLocation=15ed1af4400d,42563,1732606030451 2024-11-26T07:27:25,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] regionserver.HRegion(7855): Warmup {ENCODED => 28da289971d201b5d537d47956344363, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:25,688 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE because future has completed 2024-11-26T07:27:25,689 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf1 2024-11-26T07:27:25,689 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:25,689 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T07:27:25,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,42563,1732606030451}] 2024-11-26T07:27:25,690 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:25,690 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,691 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf2 2024-11-26T07:27:25,691 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:25,692 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] regionserver.HRegion(1722): Closing 28da289971d201b5d537d47956344363, disabling compactions & flushes 2024-11-26T07:27:25,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. after waiting 0 ms 2024-11-26T07:27:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] regionserver.HRegion(1676): Region close journal for 28da289971d201b5d537d47956344363: Waiting for close lock at 1732606045692Disabling compacts and flushes for region at 1732606045692Disabling writes for close at 1732606045692Writing region close event to WAL at 1732606045693 (+1 ms)Closed at 1732606045693 2024-11-26T07:27:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-26T07:27:25,849 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,849 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T07:27:25,850 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 28da289971d201b5d537d47956344363, disabling compactions & flushes 2024-11-26T07:27:25,850 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,850 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,850 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. after waiting 0 ms 2024-11-26T07:27:25,850 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,850 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 28da289971d201b5d537d47956344363 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-26T07:27:25,868 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf1/a266890f6be748e19b60edb48c25c10f is 35, key is r1/cf1:q/1732606045666/Put/seqid=0 2024-11-26T07:27:25,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741887_1065 (size=4783) 2024-11-26T07:27:25,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741887_1065 (size=4783) 2024-11-26T07:27:25,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741887_1065 (size=4783) 2024-11-26T07:27:25,875 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf1/a266890f6be748e19b60edb48c25c10f 2024-11-26T07:27:25,882 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf1/a266890f6be748e19b60edb48c25c10f as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/a266890f6be748e19b60edb48c25c10f 2024-11-26T07:27:25,888 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/a266890f6be748e19b60edb48c25c10f, entries=1, sequenceid=5, filesize=4.7 K 2024-11-26T07:27:25,889 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 28da289971d201b5d537d47956344363 in 39ms, sequenceid=5, compaction requested=false 2024-11-26T07:27:25,889 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-26T07:27:25,894 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-26T07:27:25,897 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:25,897 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 28da289971d201b5d537d47956344363: Waiting for close lock at 1732606045850Running coprocessor pre-close hooks at 1732606045850Disabling compacts and flushes for region at 1732606045850Disabling writes for close at 1732606045850Obtaining lock to block concurrent updates at 1732606045850Preparing flush snapshotting stores in 28da289971d201b5d537d47956344363 at 1732606045850Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732606045851 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. at 1732606045852 (+1 ms)Flushing 28da289971d201b5d537d47956344363/cf1: creating writer at 1732606045852Flushing 28da289971d201b5d537d47956344363/cf1: appending metadata at 1732606045868 (+16 ms)Flushing 28da289971d201b5d537d47956344363/cf1: closing flushed file at 1732606045868Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@467a163f: reopening flushed file at 1732606045881 (+13 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 28da289971d201b5d537d47956344363 in 39ms, sequenceid=5, compaction requested=false at 1732606045889 (+8 ms)Writing region close event to WAL at 1732606045891 (+2 ms)Running coprocessor post-close hooks at 1732606045894 (+3 ms)Closed at 1732606045897 (+3 ms) 2024-11-26T07:27:25,897 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 28da289971d201b5d537d47956344363 move to 15ed1af4400d,38495,1732606030549 record at close sequenceid=5 2024-11-26T07:27:25,900 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 28da289971d201b5d537d47956344363 2024-11-26T07:27:25,901 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=CLOSED 2024-11-26T07:27:25,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,42563,1732606030451 because future has completed 2024-11-26T07:27:25,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T07:27:25,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,42563,1732606030451 in 215 msec 2024-11-26T07:27:25,909 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE; state=CLOSED, location=15ed1af4400d,38495,1732606030549; forceNewPlan=false, retain=false 2024-11-26T07:27:26,059 INFO [15ed1af4400d:45945 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T07:27:26,060 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=OPENING, regionLocation=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:26,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE because future has completed 2024-11-26T07:27:26,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,38495,1732606030549}] 2024-11-26T07:27:26,221 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,222 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 28da289971d201b5d537d47956344363, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:26,222 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,222 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:26,222 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,222 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,224 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,225 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf1 2024-11-26T07:27:26,225 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:26,233 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/a266890f6be748e19b60edb48c25c10f 2024-11-26T07:27:26,233 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:26,233 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,234 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf2 2024-11-26T07:27:26,234 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:26,234 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:26,235 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,235 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363 2024-11-26T07:27:26,236 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363 2024-11-26T07:27:26,237 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,237 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,238 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T07:27:26,239 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,240 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 28da289971d201b5d537d47956344363; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72059101, jitterRate=0.07376427948474884}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T07:27:26,240 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,241 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 28da289971d201b5d537d47956344363: Running coprocessor pre-open hook at 1732606046223Writing region info on filesystem at 1732606046223Initializing all the Stores at 1732606046224 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606046224Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606046224Cleaning up temporary data from old regions at 1732606046237 (+13 ms)Running coprocessor post-open hooks at 1732606046240 (+3 ms)Region opened successfully at 1732606046241 (+1 ms) 2024-11-26T07:27:26,242 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., pid=9, masterSystemTime=1732606046217 2024-11-26T07:27:26,245 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,245 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=OPEN, openSeqNum=9, regionLocation=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:26,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,38495,1732606030549 because future has completed 2024-11-26T07:27:26,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-26T07:27:26,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,38495,1732606030549 in 187 msec 2024-11-26T07:27:26,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE in 570 msec 2024-11-26T07:27:26,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T07:27:26,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T07:27:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:37294 deadline: 1732606106296, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=15ed1af4400d port=38495 startCode=1732606030549. As of locationSeqNum=5. 2024-11-26T07:27:26,319 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=15ed1af4400d port=38495 startCode=1732606030549. As of locationSeqNum=5. 2024-11-26T07:27:26,320 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=15ed1af4400d port=38495 startCode=1732606030549. As of locationSeqNum=5. 2024-11-26T07:27:26,320 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,38495,1732606030549, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=15ed1af4400d port=38495 startCode=1732606030549. As of locationSeqNum=5. 2024-11-26T07:27:26,442 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28da289971d201b5d537d47956344363 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-26T07:27:26,459 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf1/f418e2086d2a431b8b41ce4cf2288389 is 29, key is r1/cf1:/1732606046432/DeleteFamily/seqid=0 2024-11-26T07:27:26,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741888_1066 (size=4906) 2024-11-26T07:27:26,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741888_1066 (size=4906) 2024-11-26T07:27:26,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741888_1066 (size=4906) 2024-11-26T07:27:26,472 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf1/f418e2086d2a431b8b41ce4cf2288389 2024-11-26T07:27:26,478 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f418e2086d2a431b8b41ce4cf2288389 2024-11-26T07:27:26,494 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf2/da69bf1cc17242e9b0a20d3420aaee41 is 29, key is r1/cf2:/1732606046432/DeleteFamily/seqid=0 2024-11-26T07:27:26,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741889_1067 (size=4906) 2024-11-26T07:27:26,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741889_1067 (size=4906) 2024-11-26T07:27:26,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741889_1067 (size=4906) 2024-11-26T07:27:26,502 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf2/da69bf1cc17242e9b0a20d3420aaee41 2024-11-26T07:27:26,508 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for da69bf1cc17242e9b0a20d3420aaee41 2024-11-26T07:27:26,510 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf1/f418e2086d2a431b8b41ce4cf2288389 as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/f418e2086d2a431b8b41ce4cf2288389 2024-11-26T07:27:26,517 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f418e2086d2a431b8b41ce4cf2288389 2024-11-26T07:27:26,517 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/f418e2086d2a431b8b41ce4cf2288389, entries=1, sequenceid=12, filesize=4.8 K 2024-11-26T07:27:26,518 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf2/da69bf1cc17242e9b0a20d3420aaee41 as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/da69bf1cc17242e9b0a20d3420aaee41 2024-11-26T07:27:26,524 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for da69bf1cc17242e9b0a20d3420aaee41 2024-11-26T07:27:26,524 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/da69bf1cc17242e9b0a20d3420aaee41, entries=1, sequenceid=12, filesize=4.8 K 2024-11-26T07:27:26,526 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 28da289971d201b5d537d47956344363 in 83ms, sequenceid=12, compaction requested=false 2024-11-26T07:27:26,526 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 28da289971d201b5d537d47956344363: 2024-11-26T07:27:26,528 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T07:27:26,530 DEBUG [Time-limited test {}] regionserver.HStore(1541): 28da289971d201b5d537d47956344363/cf1 is initiating major compaction (all files) 2024-11-26T07:27:26,530 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T07:27:26,530 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:26,531 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 28da289971d201b5d537d47956344363/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,531 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/a266890f6be748e19b60edb48c25c10f, hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/f418e2086d2a431b8b41ce4cf2288389] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp, totalSize=9.5 K 2024-11-26T07:27:26,533 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a266890f6be748e19b60edb48c25c10f, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732606045666 2024-11-26T07:27:26,533 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f418e2086d2a431b8b41ce4cf2288389, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-26T07:27:26,546 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 28da289971d201b5d537d47956344363#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T07:27:26,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741890_1068 (size=4626) 2024-11-26T07:27:26,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741890_1068 (size=4626) 2024-11-26T07:27:26,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741890_1068 (size=4626) 2024-11-26T07:27:26,561 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf1/63d2c822252c48baa3027a364f40f984 as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/63d2c822252c48baa3027a364f40f984 2024-11-26T07:27:26,576 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 28da289971d201b5d537d47956344363/cf1 of 28da289971d201b5d537d47956344363 into 63d2c822252c48baa3027a364f40f984(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T07:27:26,576 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 28da289971d201b5d537d47956344363: 2024-11-26T07:27:26,576 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-26T07:27:26,576 DEBUG [Time-limited test {}] regionserver.HStore(1541): 28da289971d201b5d537d47956344363/cf2 is initiating major compaction (all files) 2024-11-26T07:27:26,577 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T07:27:26,577 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:27:26,577 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 28da289971d201b5d537d47956344363/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,577 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/da69bf1cc17242e9b0a20d3420aaee41] into tmpdir=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp, totalSize=4.8 K 2024-11-26T07:27:26,578 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting da69bf1cc17242e9b0a20d3420aaee41, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-26T07:27:26,584 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 28da289971d201b5d537d47956344363#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T07:27:26,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741891_1069 (size=4592) 2024-11-26T07:27:26,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741891_1069 (size=4592) 2024-11-26T07:27:26,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741891_1069 (size=4592) 2024-11-26T07:27:26,598 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/.tmp/cf2/0c89089b00a140c0a7e1c443cd8d7d66 as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/0c89089b00a140c0a7e1c443cd8d7d66 2024-11-26T07:27:26,605 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 28da289971d201b5d537d47956344363/cf2 of 28da289971d201b5d537d47956344363 into 0c89089b00a140c0a7e1c443cd8d7d66(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T07:27:26,605 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 28da289971d201b5d537d47956344363: 2024-11-26T07:27:26,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=28da289971d201b5d537d47956344363, source=15ed1af4400d,38495,1732606030549, destination=15ed1af4400d,42563,1732606030451, warming up region on 15ed1af4400d,42563,1732606030451 2024-11-26T07:27:26,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=28da289971d201b5d537d47956344363, source=15ed1af4400d,38495,1732606030549, destination=15ed1af4400d,42563,1732606030451, running balancer 2024-11-26T07:27:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE 2024-11-26T07:27:26,612 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE 2024-11-26T07:27:26,613 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=CLOSING, regionLocation=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:26,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] regionserver.HRegion(7855): Warmup {ENCODED => 28da289971d201b5d537d47956344363, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:26,614 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,615 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf1 2024-11-26T07:27:26,616 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:26,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE because future has completed 2024-11-26T07:27:26,618 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T07:27:26,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,38495,1732606030549}] 2024-11-26T07:27:26,624 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/63d2c822252c48baa3027a364f40f984 2024-11-26T07:27:26,628 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/a266890f6be748e19b60edb48c25c10f 2024-11-26T07:27:26,634 INFO [StoreFileOpener-28da289971d201b5d537d47956344363-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f418e2086d2a431b8b41ce4cf2288389 2024-11-26T07:27:26,634 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/f418e2086d2a431b8b41ce4cf2288389 2024-11-26T07:27:26,634 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:26,634 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,635 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf2 2024-11-26T07:27:26,635 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:26,642 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/0c89089b00a140c0a7e1c443cd8d7d66 2024-11-26T07:27:26,647 INFO [StoreFileOpener-28da289971d201b5d537d47956344363-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for da69bf1cc17242e9b0a20d3420aaee41 2024-11-26T07:27:26,647 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/da69bf1cc17242e9b0a20d3420aaee41 2024-11-26T07:27:26,647 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] regionserver.HRegion(1722): Closing 28da289971d201b5d537d47956344363, disabling compactions & flushes 2024-11-26T07:27:26,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. after waiting 0 ms 2024-11-26T07:27:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42563 {}] regionserver.HRegion(1676): Region close journal for 28da289971d201b5d537d47956344363: Waiting for close lock at 1732606046647Disabling compacts and flushes for region at 1732606046647Disabling writes for close at 1732606046647Writing region close event to WAL at 1732606046648 (+1 ms)Closed at 1732606046648 2024-11-26T07:27:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-26T07:27:26,772 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,773 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T07:27:26,773 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 28da289971d201b5d537d47956344363, disabling compactions & flushes 2024-11-26T07:27:26,773 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,773 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,773 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. after waiting 0 ms 2024-11-26T07:27:26,773 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,773 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/a266890f6be748e19b60edb48c25c10f, hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/f418e2086d2a431b8b41ce4cf2288389] to archive 2024-11-26T07:27:26,777 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T07:27:26,781 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/a266890f6be748e19b60edb48c25c10f to hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/a266890f6be748e19b60edb48c25c10f 2024-11-26T07:27:26,783 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/f418e2086d2a431b8b41ce4cf2288389 to hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/f418e2086d2a431b8b41ce4cf2288389 2024-11-26T07:27:26,800 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/da69bf1cc17242e9b0a20d3420aaee41] to archive 2024-11-26T07:27:26,801 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T07:27:26,803 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/da69bf1cc17242e9b0a20d3420aaee41 to hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/da69bf1cc17242e9b0a20d3420aaee41 2024-11-26T07:27:26,808 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-26T07:27:26,809 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:26,809 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 28da289971d201b5d537d47956344363: Waiting for close lock at 1732606046773Running coprocessor pre-close hooks at 1732606046773Disabling compacts and flushes for region at 1732606046773Disabling writes for close at 1732606046773Writing region close event to WAL at 1732606046804 (+31 ms)Running coprocessor post-close hooks at 1732606046809 (+5 ms)Closed at 1732606046809 2024-11-26T07:27:26,809 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 28da289971d201b5d537d47956344363 move to 15ed1af4400d,42563,1732606030451 record at close sequenceid=12 2024-11-26T07:27:26,812 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 28da289971d201b5d537d47956344363 2024-11-26T07:27:26,814 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=CLOSED 2024-11-26T07:27:26,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,38495,1732606030549 because future has completed 2024-11-26T07:27:26,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-26T07:27:26,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,38495,1732606030549 in 200 msec 2024-11-26T07:27:26,824 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE; state=CLOSED, location=15ed1af4400d,42563,1732606030451; forceNewPlan=false, retain=false 2024-11-26T07:27:26,974 INFO [15ed1af4400d:45945 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T07:27:26,974 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=OPENING, regionLocation=15ed1af4400d,42563,1732606030451 2024-11-26T07:27:26,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE because future has completed 2024-11-26T07:27:26,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,42563,1732606030451}] 2024-11-26T07:27:27,134 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:27,134 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 28da289971d201b5d537d47956344363, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:27,135 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,135 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:27,135 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,135 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,137 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,138 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf1 2024-11-26T07:27:27,138 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:27,144 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/63d2c822252c48baa3027a364f40f984 2024-11-26T07:27:27,145 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:27,145 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,146 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf2 2024-11-26T07:27:27,146 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:27,152 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/0c89089b00a140c0a7e1c443cd8d7d66 2024-11-26T07:27:27,152 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:27,152 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,153 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363 2024-11-26T07:27:27,154 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363 2024-11-26T07:27:27,155 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,155 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,156 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T07:27:27,157 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,158 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 28da289971d201b5d537d47956344363; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64897457, jitterRate=-0.032952532172203064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T07:27:27,158 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,159 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 28da289971d201b5d537d47956344363: Running coprocessor pre-open hook at 1732606047135Writing region info on filesystem at 1732606047135Initializing all the Stores at 1732606047136 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606047136Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606047136Cleaning up temporary data from old regions at 1732606047155 (+19 ms)Running coprocessor post-open hooks at 1732606047158 (+3 ms)Region opened successfully at 1732606047159 (+1 ms) 2024-11-26T07:27:27,160 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., pid=12, masterSystemTime=1732606047130 2024-11-26T07:27:27,163 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:27,163 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:27,163 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=OPEN, openSeqNum=18, regionLocation=15ed1af4400d,42563,1732606030451 2024-11-26T07:27:27,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,42563,1732606030451 because future has completed 2024-11-26T07:27:27,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-26T07:27:27,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,42563,1732606030451 in 189 msec 2024-11-26T07:27:27,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, REOPEN/MOVE in 559 msec 2024-11-26T07:27:27,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T07:27:27,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T07:27:27,218 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 15ed1af4400d,42563,1732606030451: testing ***** 2024-11-26T07:27:27,218 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-26T07:27:27,221 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-26T07:27:27,223 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-26T07:27:27,227 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-26T07:27:27,228 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-26T07:27:27,238 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 278177744 }, "NonHeapMemoryUsage": { "committed": 171180032, "init": 7667712, "max": -1, "used": 168657024 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "15ed1af4400d", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "15ed1af4400d", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57836", "tag.serverName": "15ed1af4400d,42563,1732606030451", "tag.clusterId": "566e8917-4cb2-4d05-8883-249e6a806b91", "tag.Context": "regionserver", "tag.Hostname": "15ed1af4400d", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732606030451, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-26T07:27:27,242 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45945 {}] master.MasterRpcServices(700): 15ed1af4400d,42563,1732606030451 reported a fatal error: ***** ABORTING region server 15ed1af4400d,42563,1732606030451: testing ***** 2024-11-26T07:27:27,244 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '15ed1af4400d,42563,1732606030451' ***** 2024-11-26T07:27:27,244 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-26T07:27:27,245 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T07:27:27,245 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T07:27:27,245 INFO [RS:0;15ed1af4400d:42563 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-26T07:27:27,245 INFO [RS:0;15ed1af4400d:42563 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-26T07:27:27,245 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(3091): Received CLOSE for 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38495 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:33538 deadline: 1732606107245, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=15ed1af4400d port=42563 startCode=1732606030451. As of locationSeqNum=12. 2024-11-26T07:27:27,246 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(956): aborting server 15ed1af4400d,42563,1732606030451 2024-11-26T07:27:27,246 INFO [RS:0;15ed1af4400d:42563 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:27:27,246 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 28da289971d201b5d537d47956344363, disabling compactions & flushes 2024-11-26T07:27:27,246 INFO [RS:0;15ed1af4400d:42563 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;15ed1af4400d:42563. 2024-11-26T07:27:27,246 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:27,246 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,38495,1732606030549, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,38495,1732606030549, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=15ed1af4400d port=42563 startCode=1732606030451. As of locationSeqNum=12. 2024-11-26T07:27:27,246 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:27,246 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,38495,1732606030549, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=15ed1af4400d port=42563 startCode=1732606030451. As of locationSeqNum=12. 2024-11-26T07:27:27,246 DEBUG [RS:0;15ed1af4400d:42563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:27:27,246 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,38495,1732606030549, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=15ed1af4400d port=42563 startCode=1732606030451. As of locationSeqNum=12. 2024-11-26T07:27:27,246 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. after waiting 0 ms 2024-11-26T07:27:27,247 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:27,247 DEBUG [RS:0;15ed1af4400d:42563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:27:27,247 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T07:27:27,247 DEBUG [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(1325): Online Regions={28da289971d201b5d537d47956344363=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.} 2024-11-26T07:27:27,248 DEBUG [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(1351): Waiting on 28da289971d201b5d537d47956344363 2024-11-26T07:27:27,249 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:27,249 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 28da289971d201b5d537d47956344363: Waiting for close lock at 1732606047246Running coprocessor pre-close hooks at 1732606047246Disabling compacts and flushes for region at 1732606047246Disabling writes for close at 1732606047246Writing region close event to WAL at 1732606047249 (+3 ms)Running coprocessor post-close hooks at 1732606047249Closed at 1732606047249 2024-11-26T07:27:27,249 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:27,301 INFO [regionserver/15ed1af4400d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:27:27,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 15ed1af4400d,42563,1732606030451 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:27,362 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 15ed1af4400d,42563,1732606030451 aborting 2024-11-26T07:27:27,362 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 15ed1af4400d,42563,1732606030451 aborting 2024-11-26T07:27:27,362 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=12 from cache 2024-11-26T07:27:27,448 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(976): stopping server 15ed1af4400d,42563,1732606030451; all regions closed. 2024-11-26T07:27:27,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741833_1009 (size=1677) 2024-11-26T07:27:27,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741833_1009 (size=1677) 2024-11-26T07:27:27,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741833_1009 (size=1677) 2024-11-26T07:27:27,452 DEBUG [RS:0;15ed1af4400d:42563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:27:27,453 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:27:27,453 INFO [RS:0;15ed1af4400d:42563 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:27:27,453 INFO [RS:0;15ed1af4400d:42563 {}] hbase.ChoreService(370): Chore service for: regionserver/15ed1af4400d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T07:27:27,453 INFO [regionserver/15ed1af4400d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:27:27,453 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T07:27:27,453 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T07:27:27,453 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T07:27:27,453 INFO [RS:0;15ed1af4400d:42563 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:27:27,454 INFO [RS:0;15ed1af4400d:42563 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42563 2024-11-26T07:27:27,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:27:27,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/15ed1af4400d,42563,1732606030451 2024-11-26T07:27:27,460 INFO [RS:0;15ed1af4400d:42563 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:27:27,461 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [15ed1af4400d,42563,1732606030451] 2024-11-26T07:27:27,463 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/15ed1af4400d,42563,1732606030451 already deleted, retry=false 2024-11-26T07:27:27,463 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 15ed1af4400d,42563,1732606030451 on 15ed1af4400d,45945,1732606029786 2024-11-26T07:27:27,468 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 15ed1af4400d,42563,1732606030451, splitWal=true, meta=false 2024-11-26T07:27:27,471 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 15ed1af4400d,42563,1732606030451, splitWal=true, meta=false 2024-11-26T07:27:27,471 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 15ed1af4400d,42563,1732606030451 (carryingMeta=false) 15ed1af4400d,42563,1732606030451/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3f55229d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T07:27:27,472 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 15ed1af4400d,42563,1732606030451 had 1 regions 2024-11-26T07:27:27,473 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 15ed1af4400d,42563,1732606030451, splitWal=true, meta=false, isMeta: false 2024-11-26T07:27:27,475 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting 2024-11-26T07:27:27,476 INFO [PEWorker-1 {}] master.SplitWALManager(105): 15ed1af4400d,42563,1732606030451 WAL count=1, meta=false 2024-11-26T07:27:27,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 15ed1af4400d%2C42563%2C1732606030451.1732606031953}] 2024-11-26T07:27:27,483 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:27,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 15ed1af4400d%2C42563%2C1732606030451.1732606031953, worker=15ed1af4400d,38495,1732606030549}] 2024-11-26T07:27:27,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:27:27,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42563-0x101532bd7460001, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:27:27,564 INFO [RS:0;15ed1af4400d:42563 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:27:27,564 INFO [RS:0;15ed1af4400d:42563 {}] regionserver.HRegionServer(1031): Exiting; stopping=15ed1af4400d,42563,1732606030451; zookeeper connection closed. 2024-11-26T07:27:27,564 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@221bf351 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@221bf351 2024-11-26T07:27:27,573 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=18] 2024-11-26T07:27:27,575 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 15ed1af4400d:42563 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 15ed1af4400d/172.17.0.2:42563 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:27,576 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=18, error=java.net.ConnectException: Call to address=15ed1af4400d:42563 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 15ed1af4400d/172.17.0.2:42563 2024-11-26T07:27:27,576 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-26T07:27:27,576 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=18 from cache 2024-11-26T07:27:27,576 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 15ed1af4400d:42563 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 15ed1af4400d/172.17.0.2:42563 2024-11-26T07:27:27,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38495 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-26T07:27:27,663 INFO [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953, size=1.6 K (1677bytes) 2024-11-26T07:27:27,663 INFO [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953 2024-11-26T07:27:27,663 INFO [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953 after 0ms 2024-11-26T07:27:27,666 DEBUG [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:27,666 INFO [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953 took 3ms 2024-11-26T07:27:27,673 DEBUG [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 28da289971d201b5d537d47956344363: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-26T07:27:27,674 DEBUG [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953 so closing down 2024-11-26T07:27:27,674 DEBUG [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:27,674 INFO [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:27,674 INFO [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-11-26T07:27:27,674 DEBUG [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953, journal: Splitting hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953, size=1.6 K (1677bytes) at 1732606047663Finishing writing output for hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953 so closing down at 1732606047674 (+11 ms)3 split writer threads finished at 1732606047674Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1732606047674 2024-11-26T07:27:27,674 DEBUG [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953 2024-11-26T07:27:27,675 DEBUG [RS_LOG_REPLAY_OPS-regionserver/15ed1af4400d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-26T07:27:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45945 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-26T07:27:27,682 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting/15ed1af4400d%2C42563%2C1732606030451.1732606031953 to hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/oldWALs 2024-11-26T07:27:27,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-26T07:27:27,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 15ed1af4400d%2C42563%2C1732606030451.1732606031953, worker=15ed1af4400d,38495,1732606030549 in 197 msec 2024-11-26T07:27:27,686 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:27,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T07:27:27,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 15ed1af4400d%2C42563%2C1732606030451.1732606031953, worker=15ed1af4400d,38495,1732606030549 in 209 msec 2024-11-26T07:27:27,691 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting dir is empty, no logs to split. 2024-11-26T07:27:27,691 INFO [PEWorker-1 {}] master.SplitWALManager(105): 15ed1af4400d,42563,1732606030451 WAL count=0, meta=false 2024-11-26T07:27:27,691 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 15ed1af4400d,42563,1732606030451 WAL splitting is done? wals=0, meta=false 2024-11-26T07:27:27,694 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 15ed1af4400d,42563,1732606030451 failed, ignore...File hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/WALs/15ed1af4400d,42563,1732606030451-splitting does not exist. 2024-11-26T07:27:27,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, ASSIGN}] 2024-11-26T07:27:27,697 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, ASSIGN 2024-11-26T07:27:27,698 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T07:27:27,849 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(204): Hosts are {15ed1af4400d=0} racks are {/default-rack=0} 2024-11-26T07:27:27,849 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T07:27:27,849 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T07:27:27,849 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T07:27:27,849 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T07:27:27,849 INFO [15ed1af4400d:45945 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T07:27:27,849 INFO [15ed1af4400d:45945 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T07:27:27,849 DEBUG [15ed1af4400d:45945 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T07:27:27,850 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=OPENING, regionLocation=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:27,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, ASSIGN because future has completed 2024-11-26T07:27:27,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,38495,1732606030549}] 2024-11-26T07:27:27,884 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=18] 2024-11-26T07:27:27,884 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 15ed1af4400d:42563 this server is in the failed servers list 2024-11-26T07:27:27,885 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=15ed1af4400d:42563 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 15ed1af4400d:42563 2024-11-26T07:27:27,885 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 15ed1af4400d:42563 2024-11-26T07:27:27,885 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,42563,1732606030451, seqNum=18 from cache 2024-11-26T07:27:28,010 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:28,010 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 28da289971d201b5d537d47956344363, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:28,010 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 28da289971d201b5d537d47956344363 2024-11-26T07:27:28,011 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:28,011 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 28da289971d201b5d537d47956344363 2024-11-26T07:27:28,011 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 28da289971d201b5d537d47956344363 2024-11-26T07:27:28,012 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:28,013 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf1 2024-11-26T07:27:28,013 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,019 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf1/63d2c822252c48baa3027a364f40f984 2024-11-26T07:27:28,020 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,020 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 28da289971d201b5d537d47956344363 2024-11-26T07:27:28,021 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28da289971d201b5d537d47956344363 columnFamilyName cf2 2024-11-26T07:27:28,021 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,027 DEBUG [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/cf2/0c89089b00a140c0a7e1c443cd8d7d66 2024-11-26T07:27:28,027 INFO [StoreOpener-28da289971d201b5d537d47956344363-1 {}] regionserver.HStore(327): Store=28da289971d201b5d537d47956344363/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,027 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 28da289971d201b5d537d47956344363 2024-11-26T07:27:28,028 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363 2024-11-26T07:27:28,029 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363 2024-11-26T07:27:28,030 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 28da289971d201b5d537d47956344363 2024-11-26T07:27:28,030 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 28da289971d201b5d537d47956344363 2024-11-26T07:27:28,030 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T07:27:28,032 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 28da289971d201b5d537d47956344363 2024-11-26T07:27:28,033 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 28da289971d201b5d537d47956344363; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75069351, jitterRate=0.11862049996852875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T07:27:28,033 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28da289971d201b5d537d47956344363 2024-11-26T07:27:28,034 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 28da289971d201b5d537d47956344363: Running coprocessor pre-open hook at 1732606048011Writing region info on filesystem at 1732606048011Initializing all the Stores at 1732606048012 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048012Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048012Cleaning up temporary data from old regions at 1732606048030 (+18 ms)Running coprocessor post-open hooks at 1732606048033 (+3 ms)Region opened successfully at 1732606048034 (+1 ms) 2024-11-26T07:27:28,034 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., pid=17, masterSystemTime=1732606048006 2024-11-26T07:27:28,037 DEBUG [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:28,037 INFO [RS_OPEN_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:28,038 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=28da289971d201b5d537d47956344363, regionState=OPEN, openSeqNum=18, regionLocation=15ed1af4400d,38495,1732606030549 2024-11-26T07:27:28,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,38495,1732606030549 because future has completed 2024-11-26T07:27:28,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-26T07:27:28,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 28da289971d201b5d537d47956344363, server=15ed1af4400d,38495,1732606030549 in 188 msec 2024-11-26T07:27:28,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-26T07:27:28,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=28da289971d201b5d537d47956344363, ASSIGN in 348 msec 2024-11-26T07:27:28,047 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 15ed1af4400d,42563,1732606030451 after splitting done 2024-11-26T07:27:28,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 15ed1af4400d,42563,1732606030451, splitWal=true, meta=false in 583 msec 2024-11-26T07:27:28,393 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363., hostname=15ed1af4400d,38495,1732606030549, seqNum=18] 2024-11-26T07:27:28,407 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=405 (was 412), OpenFileDescriptor=1045 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=7954 (was 8002) 2024-11-26T07:27:28,409 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1045 is superior to 1024 2024-11-26T07:27:28,421 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=405, OpenFileDescriptor=1045, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=7953 2024-11-26T07:27:28,421 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1045 is superior to 1024 2024-11-26T07:27:28,436 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:28,438 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:28,439 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:28,442 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-24439934, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/hregion-24439934, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:28,456 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-24439934/hregion-24439934.1732606048442, exclude list is [], retry=0 2024-11-26T07:27:28,459 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:28,459 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:28,459 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:28,463 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-24439934/hregion-24439934.1732606048442 2024-11-26T07:27:28,463 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:28,463 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dcc53e672e8476a6aab960cf0a909504, NAME => 'testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/hbase 2024-11-26T07:27:28,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741893_1071 (size=67) 2024-11-26T07:27:28,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741893_1071 (size=67) 2024-11-26T07:27:28,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741893_1071 (size=67) 2024-11-26T07:27:28,474 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:28,475 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,477 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc53e672e8476a6aab960cf0a909504 columnFamilyName a 2024-11-26T07:27:28,477 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,477 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(327): Store=dcc53e672e8476a6aab960cf0a909504/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,477 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,479 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc53e672e8476a6aab960cf0a909504 columnFamilyName b 2024-11-26T07:27:28,479 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,479 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(327): Store=dcc53e672e8476a6aab960cf0a909504/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,479 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,481 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc53e672e8476a6aab960cf0a909504 columnFamilyName c 2024-11-26T07:27:28,481 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,481 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(327): Store=dcc53e672e8476a6aab960cf0a909504/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,481 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,482 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,483 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,484 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,484 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,484 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:28,486 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,487 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:28,488 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dcc53e672e8476a6aab960cf0a909504; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62094040, jitterRate=-0.07472670078277588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:28,489 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dcc53e672e8476a6aab960cf0a909504: Writing region info on filesystem at 1732606048474Initializing all the Stores at 1732606048475 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048475Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048475Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048475Cleaning up temporary data from old regions at 1732606048484 (+9 ms)Region opened successfully at 1732606048488 (+4 ms) 2024-11-26T07:27:28,489 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dcc53e672e8476a6aab960cf0a909504, disabling compactions & flushes 2024-11-26T07:27:28,489 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. 2024-11-26T07:27:28,489 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. 2024-11-26T07:27:28,489 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. after waiting 0 ms 2024-11-26T07:27:28,489 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. 2024-11-26T07:27:28,489 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. 2024-11-26T07:27:28,489 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dcc53e672e8476a6aab960cf0a909504: Waiting for close lock at 1732606048489Disabling compacts and flushes for region at 1732606048489Disabling writes for close at 1732606048489Writing region close event to WAL at 1732606048489Closed at 1732606048489 2024-11-26T07:27:28,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741892_1070 (size=93) 2024-11-26T07:27:28,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741892_1070 (size=93) 2024-11-26T07:27:28,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741892_1070 (size=93) 2024-11-26T07:27:28,494 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:28,494 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-24439934:(num 1732606048442) 2024-11-26T07:27:28,494 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:28,496 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:28,510 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496, exclude list is [], retry=0 2024-11-26T07:27:28,512 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:28,513 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:28,513 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:28,515 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496 2024-11-26T07:27:28,515 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:28,515 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dcc53e672e8476a6aab960cf0a909504, NAME => 'testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:28,515 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:28,515 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,515 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,517 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,518 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc53e672e8476a6aab960cf0a909504 columnFamilyName a 2024-11-26T07:27:28,518 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,519 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(327): Store=dcc53e672e8476a6aab960cf0a909504/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,519 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,519 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc53e672e8476a6aab960cf0a909504 columnFamilyName b 2024-11-26T07:27:28,520 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,520 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(327): Store=dcc53e672e8476a6aab960cf0a909504/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,520 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,521 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc53e672e8476a6aab960cf0a909504 columnFamilyName c 2024-11-26T07:27:28,521 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,521 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(327): Store=dcc53e672e8476a6aab960cf0a909504/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,522 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,522 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,524 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,525 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,525 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,526 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:28,527 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,528 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dcc53e672e8476a6aab960cf0a909504; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60154463, jitterRate=-0.10362865030765533}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:28,529 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dcc53e672e8476a6aab960cf0a909504: Writing region info on filesystem at 1732606048516Initializing all the Stores at 1732606048517 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048517Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048517Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048517Cleaning up temporary data from old regions at 1732606048525 (+8 ms)Region opened successfully at 1732606048528 (+3 ms) 2024-11-26T07:27:28,561 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dcc53e672e8476a6aab960cf0a909504 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-26T07:27:28,581 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/a/0b857915c8444567a1f2da898fc8a552 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732606048529/Put/seqid=0 2024-11-26T07:27:28,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741895_1073 (size=5958) 2024-11-26T07:27:28,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741895_1073 (size=5958) 2024-11-26T07:27:28,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741895_1073 (size=5958) 2024-11-26T07:27:28,589 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/a/0b857915c8444567a1f2da898fc8a552 2024-11-26T07:27:28,615 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/b/ae633557a9e94074be718dcfec41436d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732606048539/Put/seqid=0 2024-11-26T07:27:28,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741896_1074 (size=5958) 2024-11-26T07:27:28,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741896_1074 (size=5958) 2024-11-26T07:27:28,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741896_1074 (size=5958) 2024-11-26T07:27:28,625 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/b/ae633557a9e94074be718dcfec41436d 2024-11-26T07:27:28,652 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/c/492df1a55fa8456694bfd01b4fd7d1d6 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732606048548/Put/seqid=0 2024-11-26T07:27:28,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741897_1075 (size=5958) 2024-11-26T07:27:28,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741897_1075 (size=5958) 2024-11-26T07:27:28,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741897_1075 (size=5958) 2024-11-26T07:27:28,662 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/c/492df1a55fa8456694bfd01b4fd7d1d6 2024-11-26T07:27:28,669 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/a/0b857915c8444567a1f2da898fc8a552 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/a/0b857915c8444567a1f2da898fc8a552 2024-11-26T07:27:28,675 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/a/0b857915c8444567a1f2da898fc8a552, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T07:27:28,676 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/b/ae633557a9e94074be718dcfec41436d as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/b/ae633557a9e94074be718dcfec41436d 2024-11-26T07:27:28,682 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/b/ae633557a9e94074be718dcfec41436d, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T07:27:28,683 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/c/492df1a55fa8456694bfd01b4fd7d1d6 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/c/492df1a55fa8456694bfd01b4fd7d1d6 2024-11-26T07:27:28,690 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/c/492df1a55fa8456694bfd01b4fd7d1d6, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T07:27:28,691 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for dcc53e672e8476a6aab960cf0a909504 in 130ms, sequenceid=33, compaction requested=false 2024-11-26T07:27:28,691 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dcc53e672e8476a6aab960cf0a909504: 2024-11-26T07:27:28,691 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dcc53e672e8476a6aab960cf0a909504, disabling compactions & flushes 2024-11-26T07:27:28,692 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. 2024-11-26T07:27:28,692 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. 2024-11-26T07:27:28,692 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. after waiting 0 ms 2024-11-26T07:27:28,692 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. 2024-11-26T07:27:28,697 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. 2024-11-26T07:27:28,697 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dcc53e672e8476a6aab960cf0a909504: Waiting for close lock at 1732606048691Disabling compacts and flushes for region at 1732606048691Disabling writes for close at 1732606048692 (+1 ms)Writing region close event to WAL at 1732606048697 (+5 ms)Closed at 1732606048697 2024-11-26T07:27:28,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741894_1072 (size=2873) 2024-11-26T07:27:28,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741894_1072 (size=2873) 2024-11-26T07:27:28,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741894_1072 (size=2873) 2024-11-26T07:27:28,705 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/b/ae633557a9e94074be718dcfec41436d to hdfs://localhost:41363/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/b/ae633557a9e94074be718dcfec41436d 2024-11-26T07:27:28,729 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496, size=2.8 K (2873bytes) 2024-11-26T07:27:28,729 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496 2024-11-26T07:27:28,729 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496 after 0ms 2024-11-26T07:27:28,732 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:28,732 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496 took 3ms 2024-11-26T07:27:28,735 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496 so closing down 2024-11-26T07:27:28,735 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:28,736 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732606048496.temp 2024-11-26T07:27:28,737 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000003-wal.1732606048496.temp 2024-11-26T07:27:28,737 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:28,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741898_1076 (size=2312) 2024-11-26T07:27:28,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741898_1076 (size=2312) 2024-11-26T07:27:28,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741898_1076 (size=2312) 2024-11-26T07:27:28,745 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000003-wal.1732606048496.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:28,747 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000003-wal.1732606048496.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000032 2024-11-26T07:27:28,747 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-11-26T07:27:28,747 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496, journal: Splitting hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496, size=2.8 K (2873bytes) at 1732606048729Finishing writing output for hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496 so closing down at 1732606048735 (+6 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000003-wal.1732606048496.temp at 1732606048737 (+2 ms)3 split writer threads finished at 1732606048737Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000003-wal.1732606048496.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732606048746 (+9 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000003-wal.1732606048496.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000032 at 1732606048747 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1732606048747 2024-11-26T07:27:28,749 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048496 to hdfs://localhost:41363/hbase/oldWALs/wal.1732606048496 2024-11-26T07:27:28,750 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000032 2024-11-26T07:27:28,750 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:28,752 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:28,765 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048752, exclude list is [], retry=0 2024-11-26T07:27:28,768 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:28,768 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:28,769 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:28,770 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048752 2024-11-26T07:27:28,771 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219)] 2024-11-26T07:27:28,771 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dcc53e672e8476a6aab960cf0a909504, NAME => 'testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:28,771 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:28,771 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,771 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,772 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,773 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc53e672e8476a6aab960cf0a909504 columnFamilyName a 2024-11-26T07:27:28,773 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,778 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/a/0b857915c8444567a1f2da898fc8a552 2024-11-26T07:27:28,779 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(327): Store=dcc53e672e8476a6aab960cf0a909504/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,779 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,780 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc53e672e8476a6aab960cf0a909504 columnFamilyName b 2024-11-26T07:27:28,780 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,781 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(327): Store=dcc53e672e8476a6aab960cf0a909504/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,781 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,782 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc53e672e8476a6aab960cf0a909504 columnFamilyName c 2024-11-26T07:27:28,782 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,787 DEBUG [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/c/492df1a55fa8456694bfd01b4fd7d1d6 2024-11-26T07:27:28,787 INFO [StoreOpener-dcc53e672e8476a6aab960cf0a909504-1 {}] regionserver.HStore(327): Store=dcc53e672e8476a6aab960cf0a909504/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,787 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,788 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,789 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,790 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000032 2024-11-26T07:27:28,791 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:28,793 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000032 2024-11-26T07:27:28,793 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dcc53e672e8476a6aab960cf0a909504 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-26T07:27:28,809 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/b/8502f89dbac54235be20314b91a54fed is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732606048539/Put/seqid=0 2024-11-26T07:27:28,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741900_1078 (size=5958) 2024-11-26T07:27:28,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741900_1078 (size=5958) 2024-11-26T07:27:28,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741900_1078 (size=5958) 2024-11-26T07:27:28,817 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/b/8502f89dbac54235be20314b91a54fed 2024-11-26T07:27:28,823 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/.tmp/b/8502f89dbac54235be20314b91a54fed as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/b/8502f89dbac54235be20314b91a54fed 2024-11-26T07:27:28,827 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/b/8502f89dbac54235be20314b91a54fed, entries=10, sequenceid=32, filesize=5.8 K 2024-11-26T07:27:28,828 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for dcc53e672e8476a6aab960cf0a909504 in 35ms, sequenceid=32, compaction requested=false; wal=null 2024-11-26T07:27:28,828 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/0000000000000000032 2024-11-26T07:27:28,829 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,829 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,830 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:28,831 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dcc53e672e8476a6aab960cf0a909504 2024-11-26T07:27:28,833 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/dcc53e672e8476a6aab960cf0a909504/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-26T07:27:28,834 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dcc53e672e8476a6aab960cf0a909504; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60534113, jitterRate=-0.09797142446041107}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:28,835 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dcc53e672e8476a6aab960cf0a909504: Writing region info on filesystem at 1732606048771Initializing all the Stores at 1732606048772 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048772Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048772Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048772Obtaining lock to block concurrent updates at 1732606048793 (+21 ms)Preparing flush snapshotting stores in dcc53e672e8476a6aab960cf0a909504 at 1732606048793Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732606048793Flushing stores of testReplayEditsWrittenViaHRegion,,1732606048437.dcc53e672e8476a6aab960cf0a909504. at 1732606048793Flushing dcc53e672e8476a6aab960cf0a909504/b: creating writer at 1732606048793Flushing dcc53e672e8476a6aab960cf0a909504/b: appending metadata at 1732606048808 (+15 ms)Flushing dcc53e672e8476a6aab960cf0a909504/b: closing flushed file at 1732606048808Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60b103ae: reopening flushed file at 1732606048822 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for dcc53e672e8476a6aab960cf0a909504 in 35ms, sequenceid=32, compaction requested=false; wal=null at 1732606048828 (+6 ms)Cleaning up temporary data from old regions at 1732606048830 (+2 ms)Region opened successfully at 1732606048835 (+5 ms) 2024-11-26T07:27:28,859 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=414 (was 405) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:59282 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:59402 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:54526 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:54688 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:59196 [Waiting for operation #57] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1119 (was 1045) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 234) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7945 (was 7953) 2024-11-26T07:27:28,859 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1119 is superior to 1024 2024-11-26T07:27:28,871 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=413, OpenFileDescriptor=1119, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=7944 2024-11-26T07:27:28,871 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1119 is superior to 1024 2024-11-26T07:27:28,886 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:28,888 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:28,889 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:28,891 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-18969728, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/hregion-18969728, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:28,903 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-18969728/hregion-18969728.1732606048891, exclude list is [], retry=0 2024-11-26T07:27:28,906 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:28,906 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:28,906 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:28,908 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-18969728/hregion-18969728.1732606048891 2024-11-26T07:27:28,908 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:28,909 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 28ce3f138b13f96df7947e85b7730075, NAME => 'testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/hbase 2024-11-26T07:27:28,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741902_1080 (size=68) 2024-11-26T07:27:28,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741902_1080 (size=68) 2024-11-26T07:27:28,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741902_1080 (size=68) 2024-11-26T07:27:28,919 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:28,920 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:28,921 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ce3f138b13f96df7947e85b7730075 columnFamilyName a 2024-11-26T07:27:28,921 DEBUG [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,922 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(327): Store=28ce3f138b13f96df7947e85b7730075/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,922 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:28,923 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ce3f138b13f96df7947e85b7730075 columnFamilyName b 2024-11-26T07:27:28,923 DEBUG [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,923 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(327): Store=28ce3f138b13f96df7947e85b7730075/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,923 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:28,924 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ce3f138b13f96df7947e85b7730075 columnFamilyName c 2024-11-26T07:27:28,925 DEBUG [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:28,925 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(327): Store=28ce3f138b13f96df7947e85b7730075/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:28,925 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:28,926 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:28,926 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:28,927 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:28,927 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:28,927 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:28,928 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:28,930 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:28,931 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28ce3f138b13f96df7947e85b7730075; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66526010, jitterRate=-0.008685201406478882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:28,931 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28ce3f138b13f96df7947e85b7730075: Writing region info on filesystem at 1732606048919Initializing all the Stores at 1732606048920 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048920Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048920Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606048920Cleaning up temporary data from old regions at 1732606048927 (+7 ms)Region opened successfully at 1732606048931 (+4 ms) 2024-11-26T07:27:28,932 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28ce3f138b13f96df7947e85b7730075, disabling compactions & flushes 2024-11-26T07:27:28,932 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. 2024-11-26T07:27:28,932 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. 2024-11-26T07:27:28,932 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. after waiting 0 ms 2024-11-26T07:27:28,932 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. 2024-11-26T07:27:28,932 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. 2024-11-26T07:27:28,932 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28ce3f138b13f96df7947e85b7730075: Waiting for close lock at 1732606048932Disabling compacts and flushes for region at 1732606048932Disabling writes for close at 1732606048932Writing region close event to WAL at 1732606048932Closed at 1732606048932 2024-11-26T07:27:28,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741901_1079 (size=93) 2024-11-26T07:27:28,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741901_1079 (size=93) 2024-11-26T07:27:28,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741901_1079 (size=93) 2024-11-26T07:27:28,938 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:28,938 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-18969728:(num 1732606048891) 2024-11-26T07:27:28,938 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:28,940 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:28,962 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941, exclude list is [], retry=0 2024-11-26T07:27:28,964 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:28,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:28,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:28,967 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941 2024-11-26T07:27:28,968 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219)] 2024-11-26T07:27:29,027 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28ce3f138b13f96df7947e85b7730075, NAME => 'testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:29,030 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,030 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:29,030 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,030 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,032 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,034 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ce3f138b13f96df7947e85b7730075 columnFamilyName a 2024-11-26T07:27:29,034 DEBUG [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:29,035 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(327): Store=28ce3f138b13f96df7947e85b7730075/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:29,035 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,036 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ce3f138b13f96df7947e85b7730075 columnFamilyName b 2024-11-26T07:27:29,036 DEBUG [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:29,036 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(327): Store=28ce3f138b13f96df7947e85b7730075/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:29,037 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,037 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ce3f138b13f96df7947e85b7730075 columnFamilyName c 2024-11-26T07:27:29,037 DEBUG [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:29,038 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(327): Store=28ce3f138b13f96df7947e85b7730075/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:29,038 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,039 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,040 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,041 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,041 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,041 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:29,042 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,043 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28ce3f138b13f96df7947e85b7730075; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71490441, jitterRate=0.06529058516025543}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:29,043 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:29,044 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28ce3f138b13f96df7947e85b7730075: Running coprocessor pre-open hook at 1732606049030Writing region info on filesystem at 1732606049030Initializing all the Stores at 1732606049031 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606049032 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606049032Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606049032Cleaning up temporary data from old regions at 1732606049041 (+9 ms)Running coprocessor post-open hooks at 1732606049043 (+2 ms)Region opened successfully at 1732606049044 (+1 ms) 2024-11-26T07:27:29,056 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28ce3f138b13f96df7947e85b7730075 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-26T07:27:29,058 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28ce3f138b13f96df7947e85b7730075/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:30,058 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28ce3f138b13f96df7947e85b7730075/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:30,186 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-26T07:27:30,186 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-26T07:27:30,188 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-26T07:27:30,188 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-26T07:27:30,188 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-26T07:27:30,188 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-26T07:27:30,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-26T07:27:30,189 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-26T07:27:30,190 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-26T07:27:30,190 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-26T07:27:30,907 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T07:27:31,059 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28ce3f138b13f96df7947e85b7730075/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:32,060 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28ce3f138b13f96df7947e85b7730075/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:33,060 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28ce3f138b13f96df7947e85b7730075/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:34,061 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28ce3f138b13f96df7947e85b7730075/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:35,062 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28ce3f138b13f96df7947e85b7730075/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:36,062 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28ce3f138b13f96df7947e85b7730075/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:37,063 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28ce3f138b13f96df7947e85b7730075/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:38,063 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 28ce3f138b13f96df7947e85b7730075/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:38,065 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 28ce3f138b13f96df7947e85b7730075: 2024-11-26T07:27:38,065 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. 2024-11-26T07:27:38,077 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 28ce3f138b13f96df7947e85b7730075: 2024-11-26T07:27:38,077 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-26T07:27:38,077 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28ce3f138b13f96df7947e85b7730075, disabling compactions & flushes 2024-11-26T07:27:38,077 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. 2024-11-26T07:27:38,077 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. 2024-11-26T07:27:38,077 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. after waiting 0 ms 2024-11-26T07:27:38,077 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. 2024-11-26T07:27:38,078 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. 2024-11-26T07:27:38,078 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. 2024-11-26T07:27:38,078 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28ce3f138b13f96df7947e85b7730075: Waiting for close lock at 1732606058077Running coprocessor pre-close hooks at 1732606058077Disabling compacts and flushes for region at 1732606058077Disabling writes for close at 1732606058077Writing region close event to WAL at 1732606058078 (+1 ms)Running coprocessor post-close hooks at 1732606058078Closed at 1732606058078 2024-11-26T07:27:38,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741903_1081 (size=2398) 2024-11-26T07:27:38,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741903_1081 (size=2398) 2024-11-26T07:27:38,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741903_1081 (size=2398) 2024-11-26T07:27:38,096 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941, size=2.3 K (2398bytes) 2024-11-26T07:27:38,096 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941 2024-11-26T07:27:38,097 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941 after 1ms 2024-11-26T07:27:38,099 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:38,099 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941 took 3ms 2024-11-26T07:27:38,101 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941 so closing down 2024-11-26T07:27:38,101 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:38,102 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732606048941.temp 2024-11-26T07:27:38,103 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000004-wal.1732606048941.temp 2024-11-26T07:27:38,104 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:38,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741904_1082 (size=1672) 2024-11-26T07:27:38,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741904_1082 (size=1672) 2024-11-26T07:27:38,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741904_1082 (size=1672) 2024-11-26T07:27:38,116 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000004-wal.1732606048941.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:38,117 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000004-wal.1732606048941.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000026 2024-11-26T07:27:38,118 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-11-26T07:27:38,118 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941, journal: Splitting hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941, size=2.3 K (2398bytes) at 1732606058096Finishing writing output for hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941 so closing down at 1732606058101 (+5 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000004-wal.1732606048941.temp at 1732606058103 (+2 ms)3 split writer threads finished at 1732606058104 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000004-wal.1732606048941.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732606058116 (+12 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000004-wal.1732606048941.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000026 at 1732606058117 (+1 ms)Processed 23 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1732606058118 (+1 ms) 2024-11-26T07:27:38,120 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606048941 to hdfs://localhost:41363/hbase/oldWALs/wal.1732606048941 2024-11-26T07:27:38,121 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000026 2024-11-26T07:27:38,121 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:38,123 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:38,136 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606058123, exclude list is [], retry=0 2024-11-26T07:27:38,138 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:38,139 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:38,139 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:38,141 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606058123 2024-11-26T07:27:38,141 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:38,142 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28ce3f138b13f96df7947e85b7730075, NAME => 'testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:38,142 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,142 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:38,142 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,142 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,144 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,145 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ce3f138b13f96df7947e85b7730075 columnFamilyName a 2024-11-26T07:27:38,145 DEBUG [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,146 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(327): Store=28ce3f138b13f96df7947e85b7730075/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,146 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,147 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ce3f138b13f96df7947e85b7730075 columnFamilyName b 2024-11-26T07:27:38,147 DEBUG [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,147 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(327): Store=28ce3f138b13f96df7947e85b7730075/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,147 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,148 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28ce3f138b13f96df7947e85b7730075 columnFamilyName c 2024-11-26T07:27:38,148 DEBUG [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,148 INFO [StoreOpener-28ce3f138b13f96df7947e85b7730075-1 {}] regionserver.HStore(327): Store=28ce3f138b13f96df7947e85b7730075/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,148 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,149 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,150 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,151 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000026 2024-11-26T07:27:38,153 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:38,155 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000026 2024-11-26T07:27:38,155 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28ce3f138b13f96df7947e85b7730075 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-26T07:27:38,169 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/.tmp/a/4e6208b0cab44ed29fa414a601c941ee is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732606058070/Put/seqid=0 2024-11-26T07:27:38,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741906_1084 (size=5523) 2024-11-26T07:27:38,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741906_1084 (size=5523) 2024-11-26T07:27:38,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741906_1084 (size=5523) 2024-11-26T07:27:38,176 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/.tmp/a/4e6208b0cab44ed29fa414a601c941ee 2024-11-26T07:27:38,201 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/.tmp/b/8b66e8387e704cf8a08fc1ee52521e7d is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732606058065/Put/seqid=0 2024-11-26T07:27:38,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741907_1085 (size=5524) 2024-11-26T07:27:38,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741907_1085 (size=5524) 2024-11-26T07:27:38,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741907_1085 (size=5524) 2024-11-26T07:27:38,209 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/.tmp/b/8b66e8387e704cf8a08fc1ee52521e7d 2024-11-26T07:27:38,231 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/.tmp/c/ebd5776f2c724454b428a6c3c5f890ba is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732606058068/Put/seqid=0 2024-11-26T07:27:38,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741908_1086 (size=5457) 2024-11-26T07:27:38,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741908_1086 (size=5457) 2024-11-26T07:27:38,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741908_1086 (size=5457) 2024-11-26T07:27:38,238 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/.tmp/c/ebd5776f2c724454b428a6c3c5f890ba 2024-11-26T07:27:38,245 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/.tmp/a/4e6208b0cab44ed29fa414a601c941ee as hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/a/4e6208b0cab44ed29fa414a601c941ee 2024-11-26T07:27:38,250 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/a/4e6208b0cab44ed29fa414a601c941ee, entries=7, sequenceid=26, filesize=5.4 K 2024-11-26T07:27:38,250 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/.tmp/b/8b66e8387e704cf8a08fc1ee52521e7d as hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/b/8b66e8387e704cf8a08fc1ee52521e7d 2024-11-26T07:27:38,255 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/b/8b66e8387e704cf8a08fc1ee52521e7d, entries=7, sequenceid=26, filesize=5.4 K 2024-11-26T07:27:38,256 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/.tmp/c/ebd5776f2c724454b428a6c3c5f890ba as hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/c/ebd5776f2c724454b428a6c3c5f890ba 2024-11-26T07:27:38,261 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/c/ebd5776f2c724454b428a6c3c5f890ba, entries=6, sequenceid=26, filesize=5.3 K 2024-11-26T07:27:38,261 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 28ce3f138b13f96df7947e85b7730075 in 106ms, sequenceid=26, compaction requested=false; wal=null 2024-11-26T07:27:38,262 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/0000000000000000026 2024-11-26T07:27:38,263 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,263 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,263 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:38,265 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,267 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsAfterAbortingFlush/28ce3f138b13f96df7947e85b7730075/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-26T07:27:38,268 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28ce3f138b13f96df7947e85b7730075; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71625521, jitterRate=0.06730343401432037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:38,268 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28ce3f138b13f96df7947e85b7730075 2024-11-26T07:27:38,268 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28ce3f138b13f96df7947e85b7730075: Running coprocessor pre-open hook at 1732606058143Writing region info on filesystem at 1732606058143Initializing all the Stores at 1732606058143Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058144 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058144Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058144Obtaining lock to block concurrent updates at 1732606058155 (+11 ms)Preparing flush snapshotting stores in 28ce3f138b13f96df7947e85b7730075 at 1732606058155Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732606058155Flushing stores of testReplayEditsAfterAbortingFlush,,1732606048887.28ce3f138b13f96df7947e85b7730075. at 1732606058155Flushing 28ce3f138b13f96df7947e85b7730075/a: creating writer at 1732606058155Flushing 28ce3f138b13f96df7947e85b7730075/a: appending metadata at 1732606058169 (+14 ms)Flushing 28ce3f138b13f96df7947e85b7730075/a: closing flushed file at 1732606058169Flushing 28ce3f138b13f96df7947e85b7730075/b: creating writer at 1732606058181 (+12 ms)Flushing 28ce3f138b13f96df7947e85b7730075/b: appending metadata at 1732606058200 (+19 ms)Flushing 28ce3f138b13f96df7947e85b7730075/b: closing flushed file at 1732606058201 (+1 ms)Flushing 28ce3f138b13f96df7947e85b7730075/c: creating writer at 1732606058214 (+13 ms)Flushing 28ce3f138b13f96df7947e85b7730075/c: appending metadata at 1732606058231 (+17 ms)Flushing 28ce3f138b13f96df7947e85b7730075/c: closing flushed file at 1732606058231Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d036242: reopening flushed file at 1732606058244 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4a62d2: reopening flushed file at 1732606058250 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15d17b3d: reopening flushed file at 1732606058256 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 28ce3f138b13f96df7947e85b7730075 in 106ms, sequenceid=26, compaction requested=false; wal=null at 1732606058261 (+5 ms)Cleaning up temporary data from old regions at 1732606058263 (+2 ms)Running coprocessor post-open hooks at 1732606058268 (+5 ms)Region opened successfully at 1732606058268 2024-11-26T07:27:38,289 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=414 (was 413) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:38344 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:38626 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:49688 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:38354 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:49676 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:38590 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1179 (was 1119) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 248), ProcessCount=11 (was 11), AvailableMemoryMB=7931 (was 7944) 2024-11-26T07:27:38,289 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1179 is superior to 1024 2024-11-26T07:27:38,302 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=414, OpenFileDescriptor=1179, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=11, AvailableMemoryMB=7931 2024-11-26T07:27:38,302 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1179 is superior to 1024 2024-11-26T07:27:38,318 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:38,319 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:38,320 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:38,323 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-64210559, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/hregion-64210559, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:38,336 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-64210559/hregion-64210559.1732606058323, exclude list is [], retry=0 2024-11-26T07:27:38,339 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:38,339 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:38,339 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:38,342 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-64210559/hregion-64210559.1732606058323 2024-11-26T07:27:38,342 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:38,343 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ab3df8771ac6ab8c9e1d8b3f11c1d624, NAME => 'testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/hbase 2024-11-26T07:27:38,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741910_1088 (size=61) 2024-11-26T07:27:38,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741910_1088 (size=61) 2024-11-26T07:27:38,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741910_1088 (size=61) 2024-11-26T07:27:38,352 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:38,353 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,354 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab3df8771ac6ab8c9e1d8b3f11c1d624 columnFamilyName a 2024-11-26T07:27:38,354 DEBUG [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,355 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] regionserver.HStore(327): Store=ab3df8771ac6ab8c9e1d8b3f11c1d624/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,355 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,356 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,356 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,357 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,357 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,358 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,360 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:38,361 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab3df8771ac6ab8c9e1d8b3f11c1d624; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67144869, jitterRate=5.365163087844849E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:27:38,362 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab3df8771ac6ab8c9e1d8b3f11c1d624: Writing region info on filesystem at 1732606058352Initializing all the Stores at 1732606058353 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058353Cleaning up temporary data from old regions at 1732606058357 (+4 ms)Region opened successfully at 1732606058362 (+5 ms) 2024-11-26T07:27:38,362 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab3df8771ac6ab8c9e1d8b3f11c1d624, disabling compactions & flushes 2024-11-26T07:27:38,362 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. 2024-11-26T07:27:38,362 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. 2024-11-26T07:27:38,363 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. after waiting 0 ms 2024-11-26T07:27:38,363 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. 2024-11-26T07:27:38,363 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. 2024-11-26T07:27:38,363 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab3df8771ac6ab8c9e1d8b3f11c1d624: Waiting for close lock at 1732606058362Disabling compacts and flushes for region at 1732606058362Disabling writes for close at 1732606058363 (+1 ms)Writing region close event to WAL at 1732606058363Closed at 1732606058363 2024-11-26T07:27:38,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741909_1087 (size=93) 2024-11-26T07:27:38,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741909_1087 (size=93) 2024-11-26T07:27:38,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741909_1087 (size=93) 2024-11-26T07:27:38,368 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:38,368 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-64210559:(num 1732606058323) 2024-11-26T07:27:38,368 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:38,370 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:38,385 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370, exclude list is [], retry=0 2024-11-26T07:27:38,388 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:38,388 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:38,388 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:38,390 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370 2024-11-26T07:27:38,390 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:38,390 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ab3df8771ac6ab8c9e1d8b3f11c1d624, NAME => 'testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:38,390 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:38,390 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,390 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,391 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,392 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab3df8771ac6ab8c9e1d8b3f11c1d624 columnFamilyName a 2024-11-26T07:27:38,393 DEBUG [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,393 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] regionserver.HStore(327): Store=ab3df8771ac6ab8c9e1d8b3f11c1d624/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,393 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,394 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,395 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,395 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,395 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,397 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,398 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab3df8771ac6ab8c9e1d8b3f11c1d624; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59614498, jitterRate=-0.11167475581169128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:27:38,398 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab3df8771ac6ab8c9e1d8b3f11c1d624: Writing region info on filesystem at 1732606058390Initializing all the Stores at 1732606058391 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058391Cleaning up temporary data from old regions at 1732606058395 (+4 ms)Region opened successfully at 1732606058398 (+3 ms) 2024-11-26T07:27:38,406 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab3df8771ac6ab8c9e1d8b3f11c1d624, disabling compactions & flushes 2024-11-26T07:27:38,406 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. 2024-11-26T07:27:38,406 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. 2024-11-26T07:27:38,406 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. after waiting 0 ms 2024-11-26T07:27:38,406 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. 2024-11-26T07:27:38,407 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. 2024-11-26T07:27:38,407 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. 2024-11-26T07:27:38,407 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab3df8771ac6ab8c9e1d8b3f11c1d624: Waiting for close lock at 1732606058406Disabling compacts and flushes for region at 1732606058406Disabling writes for close at 1732606058406Writing region close event to WAL at 1732606058407 (+1 ms)Closed at 1732606058407 2024-11-26T07:27:38,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741911_1089 (size=838) 2024-11-26T07:27:38,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741911_1089 (size=838) 2024-11-26T07:27:38,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741911_1089 (size=838) 2024-11-26T07:27:38,425 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370, size=838 (838bytes) 2024-11-26T07:27:38,425 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370 2024-11-26T07:27:38,425 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370 after 0ms 2024-11-26T07:27:38,427 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:38,427 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370 took 2ms 2024-11-26T07:27:38,429 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370 so closing down 2024-11-26T07:27:38,429 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:38,430 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732606058370.temp 2024-11-26T07:27:38,432 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000003-wal.1732606058370.temp 2024-11-26T07:27:38,432 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:38,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741912_1090 (size=838) 2024-11-26T07:27:38,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741912_1090 (size=838) 2024-11-26T07:27:38,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741912_1090 (size=838) 2024-11-26T07:27:38,438 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000003-wal.1732606058370.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:38,440 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000003-wal.1732606058370.temp to hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000012 2024-11-26T07:27:38,440 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370, size=838, length=838, corrupted=false, cancelled=false 2024-11-26T07:27:38,440 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370, journal: Splitting hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370, size=838 (838bytes) at 1732606058425Finishing writing output for hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370 so closing down at 1732606058429 (+4 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000003-wal.1732606058370.temp at 1732606058432 (+3 ms)3 split writer threads finished at 1732606058432Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000003-wal.1732606058370.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732606058438 (+6 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000003-wal.1732606058370.temp to hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000012 at 1732606058440 (+2 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370, size=838, length=838, corrupted=false, cancelled=false at 1732606058440 2024-11-26T07:27:38,441 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058370 to hdfs://localhost:41363/hbase/oldWALs/wal.1732606058370 2024-11-26T07:27:38,442 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000012 2024-11-26T07:27:38,445 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:38,739 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:38,741 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:38,754 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058741, exclude list is [], retry=0 2024-11-26T07:27:38,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:38,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:38,758 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:38,759 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058741 2024-11-26T07:27:38,759 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219)] 2024-11-26T07:27:38,759 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ab3df8771ac6ab8c9e1d8b3f11c1d624, NAME => 'testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:38,760 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:38,760 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,760 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,762 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,764 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab3df8771ac6ab8c9e1d8b3f11c1d624 columnFamilyName a 2024-11-26T07:27:38,764 DEBUG [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,765 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] regionserver.HStore(327): Store=ab3df8771ac6ab8c9e1d8b3f11c1d624/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,765 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,766 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,767 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,768 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000012 2024-11-26T07:27:38,769 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:38,770 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000012 2024-11-26T07:27:38,770 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab3df8771ac6ab8c9e1d8b3f11c1d624 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-26T07:27:38,788 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/.tmp/a/936639aa97b64e018a7a9b352f655a27 is 79, key is testDatalossWhenInputError/a:x0/1732606058398/Put/seqid=0 2024-11-26T07:27:38,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741914_1092 (size=5808) 2024-11-26T07:27:38,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741914_1092 (size=5808) 2024-11-26T07:27:38,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741914_1092 (size=5808) 2024-11-26T07:27:38,795 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/.tmp/a/936639aa97b64e018a7a9b352f655a27 2024-11-26T07:27:38,805 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/.tmp/a/936639aa97b64e018a7a9b352f655a27 as hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/a/936639aa97b64e018a7a9b352f655a27 2024-11-26T07:27:38,812 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/a/936639aa97b64e018a7a9b352f655a27, entries=10, sequenceid=12, filesize=5.7 K 2024-11-26T07:27:38,813 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ab3df8771ac6ab8c9e1d8b3f11c1d624 in 42ms, sequenceid=12, compaction requested=false; wal=null 2024-11-26T07:27:38,814 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/0000000000000000012 2024-11-26T07:27:38,815 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,815 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,817 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,819 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-26T07:27:38,820 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab3df8771ac6ab8c9e1d8b3f11c1d624; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70587341, jitterRate=0.051833346486091614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:27:38,821 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab3df8771ac6ab8c9e1d8b3f11c1d624: Writing region info on filesystem at 1732606058760Initializing all the Stores at 1732606058762 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058762Obtaining lock to block concurrent updates at 1732606058770 (+8 ms)Preparing flush snapshotting stores in ab3df8771ac6ab8c9e1d8b3f11c1d624 at 1732606058770Finished memstore snapshotting testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732606058771 (+1 ms)Flushing stores of testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624. at 1732606058771Flushing ab3df8771ac6ab8c9e1d8b3f11c1d624/a: creating writer at 1732606058771Flushing ab3df8771ac6ab8c9e1d8b3f11c1d624/a: appending metadata at 1732606058787 (+16 ms)Flushing ab3df8771ac6ab8c9e1d8b3f11c1d624/a: closing flushed file at 1732606058787Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bb2436b: reopening flushed file at 1732606058803 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ab3df8771ac6ab8c9e1d8b3f11c1d624 in 42ms, sequenceid=12, compaction requested=false; wal=null at 1732606058813 (+10 ms)Cleaning up temporary data from old regions at 1732606058815 (+2 ms)Region opened successfully at 1732606058821 (+6 ms) 2024-11-26T07:27:38,824 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ab3df8771ac6ab8c9e1d8b3f11c1d624, NAME => 'testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:38,824 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732606058318.ab3df8771ac6ab8c9e1d8b3f11c1d624.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:38,824 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,824 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,825 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,826 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab3df8771ac6ab8c9e1d8b3f11c1d624 columnFamilyName a 2024-11-26T07:27:38,826 DEBUG [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,831 DEBUG [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/a/936639aa97b64e018a7a9b352f655a27 2024-11-26T07:27:38,831 INFO [StoreOpener-ab3df8771ac6ab8c9e1d8b3f11c1d624-1 {}] regionserver.HStore(327): Store=ab3df8771ac6ab8c9e1d8b3f11c1d624/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,831 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,832 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,833 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,834 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,834 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,836 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab3df8771ac6ab8c9e1d8b3f11c1d624 2024-11-26T07:27:38,838 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testDatalossWhenInputError/ab3df8771ac6ab8c9e1d8b3f11c1d624/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-26T07:27:38,839 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab3df8771ac6ab8c9e1d8b3f11c1d624; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60379783, jitterRate=-0.10027112066745758}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:27:38,839 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab3df8771ac6ab8c9e1d8b3f11c1d624: Writing region info on filesystem at 1732606058824Initializing all the Stores at 1732606058825 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058825Cleaning up temporary data from old regions at 1732606058834 (+9 ms)Region opened successfully at 1732606058839 (+5 ms) 2024-11-26T07:27:38,860 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=424 (was 414) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:38664 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:38626 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:49676 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-568013986-172.17.0.2-1732606026764:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:38408 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:49754 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1261 (was 1179) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=258 (was 228) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7924 (was 7931) 2024-11-26T07:27:38,860 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1261 is superior to 1024 2024-11-26T07:27:38,874 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=423, OpenFileDescriptor=1261, MaxFileDescriptor=1048576, SystemLoadAverage=258, ProcessCount=11, AvailableMemoryMB=7923 2024-11-26T07:27:38,874 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1261 is superior to 1024 2024-11-26T07:27:38,892 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:38,894 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:38,895 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:38,897 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-20044642, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/hregion-20044642, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:38,910 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-20044642/hregion-20044642.1732606058897, exclude list is [], retry=0 2024-11-26T07:27:38,912 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:38,912 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:38,913 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:38,914 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-20044642/hregion-20044642.1732606058897 2024-11-26T07:27:38,915 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:38,915 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 206ff911193c34b7ca56ff8f70368b0c, NAME => 'testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/hbase 2024-11-26T07:27:38,923 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T07:27:38,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741916_1094 (size=63) 2024-11-26T07:27:38,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741916_1094 (size=63) 2024-11-26T07:27:38,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741916_1094 (size=63) 2024-11-26T07:27:38,927 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:38,929 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,930 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206ff911193c34b7ca56ff8f70368b0c columnFamilyName a 2024-11-26T07:27:38,930 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,931 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(327): Store=206ff911193c34b7ca56ff8f70368b0c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,931 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,932 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206ff911193c34b7ca56ff8f70368b0c columnFamilyName b 2024-11-26T07:27:38,932 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,933 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(327): Store=206ff911193c34b7ca56ff8f70368b0c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,933 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,934 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206ff911193c34b7ca56ff8f70368b0c columnFamilyName c 2024-11-26T07:27:38,934 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,934 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(327): Store=206ff911193c34b7ca56ff8f70368b0c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,935 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,935 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,936 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,937 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,937 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,937 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:38,938 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,940 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:38,941 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 206ff911193c34b7ca56ff8f70368b0c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61867074, jitterRate=-0.0781087577342987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:38,941 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 206ff911193c34b7ca56ff8f70368b0c: Writing region info on filesystem at 1732606058927Initializing all the Stores at 1732606058928 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058928Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058928Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058928Cleaning up temporary data from old regions at 1732606058937 (+9 ms)Region opened successfully at 1732606058941 (+4 ms) 2024-11-26T07:27:38,941 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 206ff911193c34b7ca56ff8f70368b0c, disabling compactions & flushes 2024-11-26T07:27:38,941 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. 2024-11-26T07:27:38,941 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. 2024-11-26T07:27:38,941 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. after waiting 0 ms 2024-11-26T07:27:38,941 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. 2024-11-26T07:27:38,942 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. 2024-11-26T07:27:38,942 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 206ff911193c34b7ca56ff8f70368b0c: Waiting for close lock at 1732606058941Disabling compacts and flushes for region at 1732606058941Disabling writes for close at 1732606058941Writing region close event to WAL at 1732606058942 (+1 ms)Closed at 1732606058942 2024-11-26T07:27:38,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741915_1093 (size=93) 2024-11-26T07:27:38,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741915_1093 (size=93) 2024-11-26T07:27:38,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741915_1093 (size=93) 2024-11-26T07:27:38,947 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:38,947 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-20044642:(num 1732606058897) 2024-11-26T07:27:38,947 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:38,949 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:38,963 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949, exclude list is [], retry=0 2024-11-26T07:27:38,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:38,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:38,966 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:38,967 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 2024-11-26T07:27:38,967 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:38,967 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 206ff911193c34b7ca56ff8f70368b0c, NAME => 'testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:38,968 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:38,968 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,968 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,969 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,970 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206ff911193c34b7ca56ff8f70368b0c columnFamilyName a 2024-11-26T07:27:38,970 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,970 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(327): Store=206ff911193c34b7ca56ff8f70368b0c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,971 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,971 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206ff911193c34b7ca56ff8f70368b0c columnFamilyName b 2024-11-26T07:27:38,971 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,972 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(327): Store=206ff911193c34b7ca56ff8f70368b0c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,972 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,972 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206ff911193c34b7ca56ff8f70368b0c columnFamilyName c 2024-11-26T07:27:38,972 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:38,973 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(327): Store=206ff911193c34b7ca56ff8f70368b0c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:38,973 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,974 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,975 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,976 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,976 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,976 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:38,978 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:38,978 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 206ff911193c34b7ca56ff8f70368b0c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71989078, jitterRate=0.07272085547447205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:38,979 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 206ff911193c34b7ca56ff8f70368b0c: Writing region info on filesystem at 1732606058968Initializing all the Stores at 1732606058969 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058969Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058969Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606058969Cleaning up temporary data from old regions at 1732606058976 (+7 ms)Region opened successfully at 1732606058979 (+3 ms) 2024-11-26T07:27:38,982 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732606058982/Put/seqid=0 2024-11-26T07:27:38,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741918_1096 (size=4875) 2024-11-26T07:27:38,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741918_1096 (size=4875) 2024-11-26T07:27:38,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741918_1096 (size=4875) 2024-11-26T07:27:38,990 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732606058990/Put/seqid=0 2024-11-26T07:27:38,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741919_1097 (size=4875) 2024-11-26T07:27:38,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741919_1097 (size=4875) 2024-11-26T07:27:38,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741919_1097 (size=4875) 2024-11-26T07:27:39,002 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732606059002/Put/seqid=0 2024-11-26T07:27:39,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741920_1098 (size=4875) 2024-11-26T07:27:39,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741920_1098 (size=4875) 2024-11-26T07:27:39,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741920_1098 (size=4875) 2024-11-26T07:27:39,008 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 206ff911193c34b7ca56ff8f70368b0c/a 2024-11-26T07:27:39,012 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-26T07:27:39,012 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T07:27:39,012 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 206ff911193c34b7ca56ff8f70368b0c/a 2024-11-26T07:27:39,015 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-26T07:27:39,016 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T07:27:39,016 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 206ff911193c34b7ca56ff8f70368b0c/a 2024-11-26T07:27:39,019 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-26T07:27:39,019 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T07:27:39,019 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 206ff911193c34b7ca56ff8f70368b0c 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-26T07:27:39,035 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/.tmp/a/a6b5d353e9924fa3ace8015e30f00a40 is 55, key is testCompactedBulkLoadedFiles/a:a/1732606058979/Put/seqid=0 2024-11-26T07:27:39,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741921_1099 (size=5107) 2024-11-26T07:27:39,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741921_1099 (size=5107) 2024-11-26T07:27:39,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741921_1099 (size=5107) 2024-11-26T07:27:39,042 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/.tmp/a/a6b5d353e9924fa3ace8015e30f00a40 2024-11-26T07:27:39,048 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/.tmp/a/a6b5d353e9924fa3ace8015e30f00a40 as hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/a6b5d353e9924fa3ace8015e30f00a40 2024-11-26T07:27:39,052 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/a6b5d353e9924fa3ace8015e30f00a40, entries=1, sequenceid=4, filesize=5.0 K 2024-11-26T07:27:39,053 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 206ff911193c34b7ca56ff8f70368b0c in 34ms, sequenceid=4, compaction requested=false 2024-11-26T07:27:39,053 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 206ff911193c34b7ca56ff8f70368b0c: 2024-11-26T07:27:39,055 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/87038b47cfab46c8bfdcd342190ac589_SeqId_4_ 2024-11-26T07:27:39,056 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/80a473533ee1430b94f28a90881509e0_SeqId_4_ 2024-11-26T07:27:39,057 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/e29112357d2f442580728ee6c0b31cc7_SeqId_4_ 2024-11-26T07:27:39,057 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile0 into 206ff911193c34b7ca56ff8f70368b0c/a as hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/87038b47cfab46c8bfdcd342190ac589_SeqId_4_ - updating store file list. 2024-11-26T07:27:39,062 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 87038b47cfab46c8bfdcd342190ac589_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T07:27:39,062 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/87038b47cfab46c8bfdcd342190ac589_SeqId_4_ into 206ff911193c34b7ca56ff8f70368b0c/a 2024-11-26T07:27:39,062 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile0 into 206ff911193c34b7ca56ff8f70368b0c/a (new location: hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/87038b47cfab46c8bfdcd342190ac589_SeqId_4_) 2024-11-26T07:27:39,063 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile1 into 206ff911193c34b7ca56ff8f70368b0c/a as hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/80a473533ee1430b94f28a90881509e0_SeqId_4_ - updating store file list. 2024-11-26T07:27:39,067 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 80a473533ee1430b94f28a90881509e0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T07:27:39,067 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/80a473533ee1430b94f28a90881509e0_SeqId_4_ into 206ff911193c34b7ca56ff8f70368b0c/a 2024-11-26T07:27:39,067 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile1 into 206ff911193c34b7ca56ff8f70368b0c/a (new location: hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/80a473533ee1430b94f28a90881509e0_SeqId_4_) 2024-11-26T07:27:39,068 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile2 into 206ff911193c34b7ca56ff8f70368b0c/a as hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/e29112357d2f442580728ee6c0b31cc7_SeqId_4_ - updating store file list. 2024-11-26T07:27:39,071 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e29112357d2f442580728ee6c0b31cc7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T07:27:39,072 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/e29112357d2f442580728ee6c0b31cc7_SeqId_4_ into 206ff911193c34b7ca56ff8f70368b0c/a 2024-11-26T07:27:39,072 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41363/hbase/testCompactedBulkLoadedFiles/hfile2 into 206ff911193c34b7ca56ff8f70368b0c/a (new location: hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/e29112357d2f442580728ee6c0b31cc7_SeqId_4_) 2024-11-26T07:27:39,079 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T07:27:39,079 DEBUG [Time-limited test {}] regionserver.HStore(1541): 206ff911193c34b7ca56ff8f70368b0c/a is initiating major compaction (all files) 2024-11-26T07:27:39,079 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 206ff911193c34b7ca56ff8f70368b0c/a in testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. 2024-11-26T07:27:39,079 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/a6b5d353e9924fa3ace8015e30f00a40, hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/87038b47cfab46c8bfdcd342190ac589_SeqId_4_, hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/80a473533ee1430b94f28a90881509e0_SeqId_4_, hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/e29112357d2f442580728ee6c0b31cc7_SeqId_4_] into tmpdir=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/.tmp, totalSize=19.3 K 2024-11-26T07:27:39,080 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a6b5d353e9924fa3ace8015e30f00a40, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732606058979 2024-11-26T07:27:39,080 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 87038b47cfab46c8bfdcd342190ac589_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T07:27:39,081 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 80a473533ee1430b94f28a90881509e0_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T07:27:39,081 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e29112357d2f442580728ee6c0b31cc7_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T07:27:39,093 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/.tmp/a/9268b11979714c46bb8b6d71750edd80 is 55, key is testCompactedBulkLoadedFiles/a:a/1732606058979/Put/seqid=0 2024-11-26T07:27:39,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741922_1100 (size=6154) 2024-11-26T07:27:39,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741922_1100 (size=6154) 2024-11-26T07:27:39,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741922_1100 (size=6154) 2024-11-26T07:27:39,106 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/.tmp/a/9268b11979714c46bb8b6d71750edd80 as hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/9268b11979714c46bb8b6d71750edd80 2024-11-26T07:27:39,112 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 206ff911193c34b7ca56ff8f70368b0c/a of 206ff911193c34b7ca56ff8f70368b0c into 9268b11979714c46bb8b6d71750edd80(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T07:27:39,112 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 206ff911193c34b7ca56ff8f70368b0c: 2024-11-26T07:27:39,112 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-26T07:27:39,112 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-26T07:27:39,146 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949, size=0 (0bytes) 2024-11-26T07:27:39,146 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 might be still open, length is 0 2024-11-26T07:27:39,146 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 2024-11-26T07:27:39,147 WARN [IPC Server handler 1 on default port 41363 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-26T07:27:39,147 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 after 1ms 2024-11-26T07:27:40,186 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-26T07:27:40,186 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-26T07:27:40,187 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-26T07:27:40,187 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-26T07:27:41,705 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:49778 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:42007:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49778 dst: /127.0.0.1:42007 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42007 remote=/127.0.0.1:49778]. Total timeout mills is 60000, 57407 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:41,705 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:38708 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40873:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38708 dst: /127.0.0.1:40873 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:41,705 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:38448 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38448 dst: /127.0.0.1:41787 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:41,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741917_1101 (size=1368) 2024-11-26T07:27:41,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741917_1101 (size=1368) 2024-11-26T07:27:43,148 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 after 4002ms 2024-11-26T07:27:43,151 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:43,151 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 took 4005ms 2024-11-26T07:27:43,153 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949; continuing. 2024-11-26T07:27:43,153 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 so closing down 2024-11-26T07:27:43,153 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:43,155 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732606058949.temp 2024-11-26T07:27:43,157 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000003-wal.1732606058949.temp 2024-11-26T07:27:43,157 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:43,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741923_1102 (size=635) 2024-11-26T07:27:43,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741923_1102 (size=635) 2024-11-26T07:27:43,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741923_1102 (size=635) 2024-11-26T07:27:43,169 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000003-wal.1732606058949.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:43,171 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000003-wal.1732606058949.temp to hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000008 2024-11-26T07:27:43,171 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T07:27:43,171 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949, journal: Splitting hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949, size=0 (0bytes) at 1732606059146Finishing writing output for hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 so closing down at 1732606063153 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000003-wal.1732606058949.temp at 1732606063157 (+4 ms)3 split writer threads finished at 1732606063157Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000003-wal.1732606058949.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732606063169 (+12 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000003-wal.1732606058949.temp to hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000008 at 1732606063171 (+2 ms)Processed 5 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949, size=0, length=0, corrupted=false, cancelled=false at 1732606063171 2024-11-26T07:27:43,173 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 to hdfs://localhost:41363/hbase/oldWALs/wal.1732606058949 2024-11-26T07:27:43,174 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000008 2024-11-26T07:27:43,174 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:43,176 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:43,193 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606063176, exclude list is [], retry=0 2024-11-26T07:27:43,196 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:43,196 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:43,197 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:43,198 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606063176 2024-11-26T07:27:43,198 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:43,198 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 206ff911193c34b7ca56ff8f70368b0c, NAME => 'testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:43,199 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:43,199 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,199 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,200 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,201 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206ff911193c34b7ca56ff8f70368b0c columnFamilyName a 2024-11-26T07:27:43,201 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,208 DEBUG [StoreFileOpener-206ff911193c34b7ca56ff8f70368b0c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 80a473533ee1430b94f28a90881509e0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T07:27:43,208 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/80a473533ee1430b94f28a90881509e0_SeqId_4_ 2024-11-26T07:27:43,212 DEBUG [StoreFileOpener-206ff911193c34b7ca56ff8f70368b0c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 87038b47cfab46c8bfdcd342190ac589_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T07:27:43,212 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/87038b47cfab46c8bfdcd342190ac589_SeqId_4_ 2024-11-26T07:27:43,217 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/9268b11979714c46bb8b6d71750edd80 2024-11-26T07:27:43,221 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/a6b5d353e9924fa3ace8015e30f00a40 2024-11-26T07:27:43,225 DEBUG [StoreFileOpener-206ff911193c34b7ca56ff8f70368b0c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e29112357d2f442580728ee6c0b31cc7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T07:27:43,225 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/e29112357d2f442580728ee6c0b31cc7_SeqId_4_ 2024-11-26T07:27:43,226 WARN [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/80a473533ee1430b94f28a90881509e0_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@73ba15fb 2024-11-26T07:27:43,226 WARN [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/87038b47cfab46c8bfdcd342190ac589_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@73ba15fb 2024-11-26T07:27:43,226 WARN [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/a6b5d353e9924fa3ace8015e30f00a40 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@73ba15fb 2024-11-26T07:27:43,226 WARN [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/e29112357d2f442580728ee6c0b31cc7_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@73ba15fb 2024-11-26T07:27:43,226 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/80a473533ee1430b94f28a90881509e0_SeqId_4_, hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/87038b47cfab46c8bfdcd342190ac589_SeqId_4_, hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/a6b5d353e9924fa3ace8015e30f00a40, hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/e29112357d2f442580728ee6c0b31cc7_SeqId_4_] to archive 2024-11-26T07:27:43,227 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T07:27:43,229 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/80a473533ee1430b94f28a90881509e0_SeqId_4_ to hdfs://localhost:41363/hbase/archive/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/80a473533ee1430b94f28a90881509e0_SeqId_4_ 2024-11-26T07:27:43,230 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/87038b47cfab46c8bfdcd342190ac589_SeqId_4_ to hdfs://localhost:41363/hbase/archive/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/87038b47cfab46c8bfdcd342190ac589_SeqId_4_ 2024-11-26T07:27:43,231 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/a6b5d353e9924fa3ace8015e30f00a40 to hdfs://localhost:41363/hbase/archive/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/a6b5d353e9924fa3ace8015e30f00a40 2024-11-26T07:27:43,233 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/e29112357d2f442580728ee6c0b31cc7_SeqId_4_ to hdfs://localhost:41363/hbase/archive/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/a/e29112357d2f442580728ee6c0b31cc7_SeqId_4_ 2024-11-26T07:27:43,233 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(327): Store=206ff911193c34b7ca56ff8f70368b0c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,233 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,234 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206ff911193c34b7ca56ff8f70368b0c columnFamilyName b 2024-11-26T07:27:43,234 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,234 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(327): Store=206ff911193c34b7ca56ff8f70368b0c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,234 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,235 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 206ff911193c34b7ca56ff8f70368b0c columnFamilyName c 2024-11-26T07:27:43,235 DEBUG [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,236 INFO [StoreOpener-206ff911193c34b7ca56ff8f70368b0c-1 {}] regionserver.HStore(327): Store=206ff911193c34b7ca56ff8f70368b0c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,236 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,237 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,239 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000008 2024-11-26T07:27:43,241 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:43,243 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 206ff911193c34b7ca56ff8f70368b0c : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "206ff911193c34b7ca56ff8f70368b0c" family_name: "a" compaction_input: "a6b5d353e9924fa3ace8015e30f00a40" compaction_input: "87038b47cfab46c8bfdcd342190ac589_SeqId_4_" compaction_input: "80a473533ee1430b94f28a90881509e0_SeqId_4_" compaction_input: "e29112357d2f442580728ee6c0b31cc7_SeqId_4_" compaction_output: "9268b11979714c46bb8b6d71750edd80" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-26T07:27:43,243 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-26T07:27:43,243 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000008 2024-11-26T07:27:43,243 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/0000000000000000008 2024-11-26T07:27:43,245 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,245 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,246 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:43,247 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 206ff911193c34b7ca56ff8f70368b0c 2024-11-26T07:27:43,249 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testCompactedBulkLoadedFiles/206ff911193c34b7ca56ff8f70368b0c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-26T07:27:43,251 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 206ff911193c34b7ca56ff8f70368b0c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68887482, jitterRate=0.02650347352027893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:43,251 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 206ff911193c34b7ca56ff8f70368b0c: Writing region info on filesystem at 1732606063199Initializing all the Stores at 1732606063200 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063200Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063200Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063200Cleaning up temporary data from old regions at 1732606063245 (+45 ms)Region opened successfully at 1732606063251 (+6 ms) 2024-11-26T07:27:43,254 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 206ff911193c34b7ca56ff8f70368b0c, disabling compactions & flushes 2024-11-26T07:27:43,254 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. 2024-11-26T07:27:43,254 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. 2024-11-26T07:27:43,254 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. after waiting 0 ms 2024-11-26T07:27:43,254 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. 2024-11-26T07:27:43,255 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732606058893.206ff911193c34b7ca56ff8f70368b0c. 2024-11-26T07:27:43,255 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 206ff911193c34b7ca56ff8f70368b0c: Waiting for close lock at 1732606063254Disabling compacts and flushes for region at 1732606063254Disabling writes for close at 1732606063254Writing region close event to WAL at 1732606063255 (+1 ms)Closed at 1732606063255 2024-11-26T07:27:43,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741924_1103 (size=93) 2024-11-26T07:27:43,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741924_1103 (size=93) 2024-11-26T07:27:43,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741924_1103 (size=93) 2024-11-26T07:27:43,260 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:43,260 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732606063176) 2024-11-26T07:27:43,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=437 (was 423) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1493698477_22 at /127.0.0.1:37014 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:41363 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:41363 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1493698477_22 at /127.0.0.1:43232 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1493698477_22 at /127.0.0.1:60674 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1261) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=258 (was 258), ProcessCount=11 (was 11), AvailableMemoryMB=7723 (was 7923) 2024-11-26T07:27:43,275 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-26T07:27:43,291 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=437, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=258, ProcessCount=11, AvailableMemoryMB=7721 2024-11-26T07:27:43,291 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-26T07:27:43,310 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:43,313 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:27:43,314 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:27:43,316 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72111225, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/hregion-72111225, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:43,333 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72111225/hregion-72111225.1732606063317, exclude list is [], retry=0 2024-11-26T07:27:43,336 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:43,336 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:43,337 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:43,344 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72111225/hregion-72111225.1732606063317 2024-11-26T07:27:43,345 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:43,346 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4d306a3b6acc53a30a267b6f55069486, NAME => 'testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41363/hbase 2024-11-26T07:27:43,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741926_1105 (size=67) 2024-11-26T07:27:43,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741926_1105 (size=67) 2024-11-26T07:27:43,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741926_1105 (size=67) 2024-11-26T07:27:43,359 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:43,360 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,362 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName a 2024-11-26T07:27:43,362 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,363 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,363 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,365 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName b 2024-11-26T07:27:43,365 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,366 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,366 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,367 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName c 2024-11-26T07:27:43,367 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,368 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,368 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,370 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,370 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,371 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:43,372 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,375 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:27:43,375 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4d306a3b6acc53a30a267b6f55069486; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60504787, jitterRate=-0.09840841591358185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:43,376 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4d306a3b6acc53a30a267b6f55069486: Writing region info on filesystem at 1732606063359Initializing all the Stores at 1732606063360 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063360Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063360Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063360Cleaning up temporary data from old regions at 1732606063370 (+10 ms)Region opened successfully at 1732606063375 (+5 ms) 2024-11-26T07:27:43,376 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4d306a3b6acc53a30a267b6f55069486, disabling compactions & flushes 2024-11-26T07:27:43,376 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:43,376 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:43,376 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. after waiting 0 ms 2024-11-26T07:27:43,376 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:43,376 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:43,376 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4d306a3b6acc53a30a267b6f55069486: Waiting for close lock at 1732606063376Disabling compacts and flushes for region at 1732606063376Disabling writes for close at 1732606063376Writing region close event to WAL at 1732606063376Closed at 1732606063376 2024-11-26T07:27:43,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741925_1104 (size=93) 2024-11-26T07:27:43,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741925_1104 (size=93) 2024-11-26T07:27:43,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741925_1104 (size=93) 2024-11-26T07:27:43,382 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:43,382 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72111225:(num 1732606063317) 2024-11-26T07:27:43,382 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:43,383 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:43,397 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384, exclude list is [], retry=0 2024-11-26T07:27:43,399 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:43,400 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:43,400 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:43,402 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384 2024-11-26T07:27:43,402 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:43,402 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d306a3b6acc53a30a267b6f55069486, NAME => 'testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:43,402 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:43,402 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,402 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,404 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,405 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName a 2024-11-26T07:27:43,405 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,405 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,405 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,406 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName b 2024-11-26T07:27:43,406 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,407 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,407 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,407 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName c 2024-11-26T07:27:43,407 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,408 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,408 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,410 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,411 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,411 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,411 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:43,413 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,414 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4d306a3b6acc53a30a267b6f55069486; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67512790, jitterRate=0.006018966436386108}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:43,414 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4d306a3b6acc53a30a267b6f55069486: Writing region info on filesystem at 1732606063403Initializing all the Stores at 1732606063403Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063403Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063404 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063404Cleaning up temporary data from old regions at 1732606063411 (+7 ms)Region opened successfully at 1732606063414 (+3 ms) 2024-11-26T07:27:43,422 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4d306a3b6acc53a30a267b6f55069486 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-26T07:27:43,439 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/a/cae85a0edf144ba8a2ff381eb6ed63f0 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732606063414/Put/seqid=0 2024-11-26T07:27:43,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741928_1107 (size=5958) 2024-11-26T07:27:43,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741928_1107 (size=5958) 2024-11-26T07:27:43,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741928_1107 (size=5958) 2024-11-26T07:27:43,450 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/a/cae85a0edf144ba8a2ff381eb6ed63f0 2024-11-26T07:27:43,456 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/a/cae85a0edf144ba8a2ff381eb6ed63f0 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/a/cae85a0edf144ba8a2ff381eb6ed63f0 2024-11-26T07:27:43,462 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/a/cae85a0edf144ba8a2ff381eb6ed63f0, entries=10, sequenceid=13, filesize=5.8 K 2024-11-26T07:27:43,464 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4d306a3b6acc53a30a267b6f55069486 in 42ms, sequenceid=13, compaction requested=false 2024-11-26T07:27:43,464 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4d306a3b6acc53a30a267b6f55069486: 2024-11-26T07:27:43,485 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4d306a3b6acc53a30a267b6f55069486, disabling compactions & flushes 2024-11-26T07:27:43,485 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:43,485 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:43,485 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. after waiting 0 ms 2024-11-26T07:27:43,485 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:43,486 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:43,487 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:43,487 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4d306a3b6acc53a30a267b6f55069486: Waiting for close lock at 1732606063485Disabling compacts and flushes for region at 1732606063485Disabling writes for close at 1732606063485Writing region close event to WAL at 1732606063486 (+1 ms)Closed at 1732606063487 (+1 ms) 2024-11-26T07:27:43,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741927_1106 (size=2805) 2024-11-26T07:27:43,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741927_1106 (size=2805) 2024-11-26T07:27:43,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741927_1106 (size=2805) 2024-11-26T07:27:43,511 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384, size=2.7 K (2805bytes) 2024-11-26T07:27:43,511 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384 2024-11-26T07:27:43,512 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384 after 1ms 2024-11-26T07:27:43,514 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:43,515 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384 took 3ms 2024-11-26T07:27:43,517 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384 so closing down 2024-11-26T07:27:43,517 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:43,518 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732606063384.temp 2024-11-26T07:27:43,519 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000003-wal.1732606063384.temp 2024-11-26T07:27:43,520 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:43,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741929_1108 (size=2312) 2024-11-26T07:27:43,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741929_1108 (size=2312) 2024-11-26T07:27:43,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741929_1108 (size=2312) 2024-11-26T07:27:43,528 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000003-wal.1732606063384.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:43,529 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000003-wal.1732606063384.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000035 2024-11-26T07:27:43,529 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-11-26T07:27:43,530 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384, journal: Splitting hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384, size=2.7 K (2805bytes) at 1732606063511Finishing writing output for hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384 so closing down at 1732606063517 (+6 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000003-wal.1732606063384.temp at 1732606063519 (+2 ms)3 split writer threads finished at 1732606063520 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000003-wal.1732606063384.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732606063528 (+8 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000003-wal.1732606063384.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000035 at 1732606063529 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1732606063529 2024-11-26T07:27:43,531 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063384 to hdfs://localhost:41363/hbase/oldWALs/wal.1732606063384 2024-11-26T07:27:43,532 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000035 2024-11-26T07:27:43,532 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:43,534 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:43,549 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535, exclude list is [], retry=0 2024-11-26T07:27:43,553 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:43,553 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:43,553 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:43,573 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 2024-11-26T07:27:43,576 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36293:36293),(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167)] 2024-11-26T07:27:43,577 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d306a3b6acc53a30a267b6f55069486, NAME => 'testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:27:43,577 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:43,577 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,577 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,579 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,580 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName a 2024-11-26T07:27:43,580 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,586 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/a/cae85a0edf144ba8a2ff381eb6ed63f0 2024-11-26T07:27:43,586 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,586 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,587 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName b 2024-11-26T07:27:43,587 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,588 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,588 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,588 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName c 2024-11-26T07:27:43,589 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:43,589 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:43,589 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,590 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,591 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,591 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000035 2024-11-26T07:27:43,593 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:43,594 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000035 2024-11-26T07:27:43,595 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4d306a3b6acc53a30a267b6f55069486 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-26T07:27:43,601 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T07:27:43,613 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/b/0b775c3d1ff846998e4f6d36819c27a5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732606063464/Put/seqid=0 2024-11-26T07:27:43,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741931_1110 (size=5958) 2024-11-26T07:27:43,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741931_1110 (size=5958) 2024-11-26T07:27:43,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741931_1110 (size=5958) 2024-11-26T07:27:43,627 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/b/0b775c3d1ff846998e4f6d36819c27a5 2024-11-26T07:27:43,649 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/c/347c08aecc7444bba3a7e6767d7e1c29 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732606063472/Put/seqid=0 2024-11-26T07:27:43,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741932_1111 (size=5958) 2024-11-26T07:27:43,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741932_1111 (size=5958) 2024-11-26T07:27:43,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741932_1111 (size=5958) 2024-11-26T07:27:43,657 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/c/347c08aecc7444bba3a7e6767d7e1c29 2024-11-26T07:27:43,663 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/b/0b775c3d1ff846998e4f6d36819c27a5 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/b/0b775c3d1ff846998e4f6d36819c27a5 2024-11-26T07:27:43,669 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/b/0b775c3d1ff846998e4f6d36819c27a5, entries=10, sequenceid=35, filesize=5.8 K 2024-11-26T07:27:43,670 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/c/347c08aecc7444bba3a7e6767d7e1c29 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/c/347c08aecc7444bba3a7e6767d7e1c29 2024-11-26T07:27:43,676 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/c/347c08aecc7444bba3a7e6767d7e1c29, entries=10, sequenceid=35, filesize=5.8 K 2024-11-26T07:27:43,676 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 4d306a3b6acc53a30a267b6f55069486 in 82ms, sequenceid=35, compaction requested=false; wal=null 2024-11-26T07:27:43,677 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000035 2024-11-26T07:27:43,678 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,678 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,679 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:43,680 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:43,682 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-26T07:27:43,683 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4d306a3b6acc53a30a267b6f55069486; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70518147, jitterRate=0.05080227553844452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:43,683 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4d306a3b6acc53a30a267b6f55069486: Writing region info on filesystem at 1732606063577Initializing all the Stores at 1732606063578 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063578Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063579 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606063579Obtaining lock to block concurrent updates at 1732606063595 (+16 ms)Preparing flush snapshotting stores in 4d306a3b6acc53a30a267b6f55069486 at 1732606063595Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732606063595Flushing stores of testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. at 1732606063595Flushing 4d306a3b6acc53a30a267b6f55069486/b: creating writer at 1732606063595Flushing 4d306a3b6acc53a30a267b6f55069486/b: appending metadata at 1732606063612 (+17 ms)Flushing 4d306a3b6acc53a30a267b6f55069486/b: closing flushed file at 1732606063612Flushing 4d306a3b6acc53a30a267b6f55069486/c: creating writer at 1732606063632 (+20 ms)Flushing 4d306a3b6acc53a30a267b6f55069486/c: appending metadata at 1732606063648 (+16 ms)Flushing 4d306a3b6acc53a30a267b6f55069486/c: closing flushed file at 1732606063648Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62dd8498: reopening flushed file at 1732606063662 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50cd4f83: reopening flushed file at 1732606063669 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 4d306a3b6acc53a30a267b6f55069486 in 82ms, sequenceid=35, compaction requested=false; wal=null at 1732606063676 (+7 ms)Cleaning up temporary data from old regions at 1732606063678 (+2 ms)Region opened successfully at 1732606063683 (+5 ms) 2024-11-26T07:27:43,747 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535, size=0 (0bytes) 2024-11-26T07:27:43,747 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 might be still open, length is 0 2024-11-26T07:27:43,747 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 2024-11-26T07:27:43,748 WARN [IPC Server handler 3 on default port 41363 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-26T07:27:43,748 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 after 1ms 2024-11-26T07:27:44,706 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:43302 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40873:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43302 dst: /127.0.0.1:40873 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40873 remote=/127.0.0.1:43302]. Total timeout mills is 60000, 59004 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:44,707 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:37078 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:42007:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37078 dst: /127.0.0.1:42007 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:44,707 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_802469026_22 at /127.0.0.1:60738 [Receiving block BP-568013986-172.17.0.2-1732606026764:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60738 dst: /127.0.0.1:41787 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:44,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741930_1112 (size=2304) 2024-11-26T07:27:44,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741930_1112 (size=2304) 2024-11-26T07:27:44,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741930_1112 (size=2304) 2024-11-26T07:27:47,749 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 after 4002ms 2024-11-26T07:27:47,751 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:47,752 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 took 4005ms 2024-11-26T07:27:47,753 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535; continuing. 2024-11-26T07:27:47,754 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 so closing down 2024-11-26T07:27:47,754 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T07:27:47,755 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732606063535.temp 2024-11-26T07:27:47,757 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000037-wal.1732606063535.temp 2024-11-26T07:27:47,757 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T07:27:47,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741933_1113 (size=2312) 2024-11-26T07:27:47,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741933_1113 (size=2312) 2024-11-26T07:27:47,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741933_1113 (size=2312) 2024-11-26T07:27:47,767 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000037-wal.1732606063535.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T07:27:47,768 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000037-wal.1732606063535.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000066 2024-11-26T07:27:47,769 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T07:27:47,769 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535, journal: Splitting hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535, size=0 (0bytes) at 1732606063747Finishing writing output for hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 so closing down at 1732606067754 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000037-wal.1732606063535.temp at 1732606067757 (+3 ms)3 split writer threads finished at 1732606067757Closed recovered edits writer path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000037-wal.1732606063535.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732606067767 (+10 ms)Rename recovered edits hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000037-wal.1732606063535.temp to hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000066 at 1732606067768 (+1 ms)Processed 30 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535, size=0, length=0, corrupted=false, cancelled=false at 1732606067769 (+1 ms) 2024-11-26T07:27:47,770 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 to hdfs://localhost:41363/hbase/oldWALs/wal.1732606063535 2024-11-26T07:27:47,771 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000066 2024-11-26T07:27:47,771 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T07:27:47,773 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41363/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309, archiveDir=hdfs://localhost:41363/hbase/oldWALs, maxLogs=32 2024-11-26T07:27:47,786 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606067773, exclude list is [], retry=0 2024-11-26T07:27:47,788 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42007,DS-cc923b65-4ccd-4e0e-b363-2549e1aea740,DISK] 2024-11-26T07:27:47,789 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41787,DS-036b023e-8ee6-41f7-ac95-88c88b652934,DISK] 2024-11-26T07:27:47,789 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40873,DS-19c396ae-f4e6-4038-b472-ee0a6d0c84be,DISK] 2024-11-26T07:27:47,790 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606067773 2024-11-26T07:27:47,791 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46219:46219),(127.0.0.1/127.0.0.1:41167:41167),(127.0.0.1/127.0.0.1:36293:36293)] 2024-11-26T07:27:47,791 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:27:47,792 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:47,793 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName a 2024-11-26T07:27:47,793 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:47,797 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/a/cae85a0edf144ba8a2ff381eb6ed63f0 2024-11-26T07:27:47,797 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:47,798 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:47,798 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName b 2024-11-26T07:27:47,798 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:47,802 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/b/0b775c3d1ff846998e4f6d36819c27a5 2024-11-26T07:27:47,802 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:47,802 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:47,803 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d306a3b6acc53a30a267b6f55069486 columnFamilyName c 2024-11-26T07:27:47,803 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:27:47,807 DEBUG [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/c/347c08aecc7444bba3a7e6767d7e1c29 2024-11-26T07:27:47,807 INFO [StoreOpener-4d306a3b6acc53a30a267b6f55069486-1 {}] regionserver.HStore(327): Store=4d306a3b6acc53a30a267b6f55069486/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:27:47,808 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:47,808 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:47,809 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:47,810 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000066 2024-11-26T07:27:47,812 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T07:27:47,817 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000066 2024-11-26T07:27:47,817 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4d306a3b6acc53a30a267b6f55069486 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-26T07:27:47,833 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/a/a352aafe5a484dea9504cc6785ee71f0 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732606063690/Put/seqid=0 2024-11-26T07:27:47,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741935_1115 (size=5958) 2024-11-26T07:27:47,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741935_1115 (size=5958) 2024-11-26T07:27:47,841 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/a/a352aafe5a484dea9504cc6785ee71f0 2024-11-26T07:27:47,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741935_1115 (size=5958) 2024-11-26T07:27:47,860 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/b/3eac4547bbbc4a988131e0fe2e99b6e9 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732606063697/Put/seqid=0 2024-11-26T07:27:47,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741936_1116 (size=5958) 2024-11-26T07:27:47,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741936_1116 (size=5958) 2024-11-26T07:27:47,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741936_1116 (size=5958) 2024-11-26T07:27:47,867 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/b/3eac4547bbbc4a988131e0fe2e99b6e9 2024-11-26T07:27:47,886 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/c/0650e265642b41cc8d112bc42dba4530 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732606063703/Put/seqid=0 2024-11-26T07:27:47,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741937_1117 (size=5958) 2024-11-26T07:27:47,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741937_1117 (size=5958) 2024-11-26T07:27:47,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741937_1117 (size=5958) 2024-11-26T07:27:47,892 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/c/0650e265642b41cc8d112bc42dba4530 2024-11-26T07:27:47,897 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/a/a352aafe5a484dea9504cc6785ee71f0 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/a/a352aafe5a484dea9504cc6785ee71f0 2024-11-26T07:27:47,901 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/a/a352aafe5a484dea9504cc6785ee71f0, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T07:27:47,902 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/b/3eac4547bbbc4a988131e0fe2e99b6e9 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/b/3eac4547bbbc4a988131e0fe2e99b6e9 2024-11-26T07:27:47,906 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/b/3eac4547bbbc4a988131e0fe2e99b6e9, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T07:27:47,907 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/.tmp/c/0650e265642b41cc8d112bc42dba4530 as hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/c/0650e265642b41cc8d112bc42dba4530 2024-11-26T07:27:47,910 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/c/0650e265642b41cc8d112bc42dba4530, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T07:27:47,911 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4d306a3b6acc53a30a267b6f55069486 in 93ms, sequenceid=66, compaction requested=false; wal=null 2024-11-26T07:27:47,911 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/0000000000000000066 2024-11-26T07:27:47,912 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:47,912 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:47,913 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T07:27:47,914 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4d306a3b6acc53a30a267b6f55069486 2024-11-26T07:27:47,916 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/hbase/data/default/testReplayEditsWrittenViaHRegion/4d306a3b6acc53a30a267b6f55069486/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-26T07:27:47,916 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4d306a3b6acc53a30a267b6f55069486; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63590382, jitterRate=-0.05242946743965149}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T07:27:47,917 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4d306a3b6acc53a30a267b6f55069486: Writing region info on filesystem at 1732606067791Initializing all the Stores at 1732606067792 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606067792Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606067792Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732606067792Obtaining lock to block concurrent updates at 1732606067817 (+25 ms)Preparing flush snapshotting stores in 4d306a3b6acc53a30a267b6f55069486 at 1732606067817Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732606067817Flushing stores of testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. at 1732606067817Flushing 4d306a3b6acc53a30a267b6f55069486/a: creating writer at 1732606067817Flushing 4d306a3b6acc53a30a267b6f55069486/a: appending metadata at 1732606067832 (+15 ms)Flushing 4d306a3b6acc53a30a267b6f55069486/a: closing flushed file at 1732606067832Flushing 4d306a3b6acc53a30a267b6f55069486/b: creating writer at 1732606067846 (+14 ms)Flushing 4d306a3b6acc53a30a267b6f55069486/b: appending metadata at 1732606067860 (+14 ms)Flushing 4d306a3b6acc53a30a267b6f55069486/b: closing flushed file at 1732606067860Flushing 4d306a3b6acc53a30a267b6f55069486/c: creating writer at 1732606067871 (+11 ms)Flushing 4d306a3b6acc53a30a267b6f55069486/c: appending metadata at 1732606067885 (+14 ms)Flushing 4d306a3b6acc53a30a267b6f55069486/c: closing flushed file at 1732606067885Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a43b910: reopening flushed file at 1732606067896 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@577172a5: reopening flushed file at 1732606067902 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3beecaf0: reopening flushed file at 1732606067906 (+4 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4d306a3b6acc53a30a267b6f55069486 in 93ms, sequenceid=66, compaction requested=false; wal=null at 1732606067911 (+5 ms)Cleaning up temporary data from old regions at 1732606067912 (+1 ms)Region opened successfully at 1732606067917 (+5 ms) 2024-11-26T07:27:47,930 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4d306a3b6acc53a30a267b6f55069486, disabling compactions & flushes 2024-11-26T07:27:47,930 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:47,930 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:47,930 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. after waiting 0 ms 2024-11-26T07:27:47,930 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:47,931 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732606063311.4d306a3b6acc53a30a267b6f55069486. 2024-11-26T07:27:47,931 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4d306a3b6acc53a30a267b6f55069486: Waiting for close lock at 1732606067930Disabling compacts and flushes for region at 1732606067930Disabling writes for close at 1732606067930Writing region close event to WAL at 1732606067931 (+1 ms)Closed at 1732606067931 2024-11-26T07:27:47,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741934_1114 (size=93) 2024-11-26T07:27:47,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741934_1114 (size=93) 2024-11-26T07:27:47,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741934_1114 (size=93) 2024-11-26T07:27:47,936 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T07:27:47,936 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732606067773) 2024-11-26T07:27:47,951 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=444 (was 437) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1826283870_22 at /127.0.0.1:43336 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:41363 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1826283870_22 at /127.0.0.1:60778 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:41363 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1826283870_22 at /127.0.0.1:37122 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 258) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7485 (was 7721) 2024-11-26T07:27:47,952 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-26T07:27:47,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T07:27:47,952 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T07:27:47,952 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:27:47,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:27:47,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:27:47,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T07:27:47,953 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T07:27:47,953 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=814710804, stopped=false 2024-11-26T07:27:47,953 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=15ed1af4400d,45945,1732606029786 2024-11-26T07:27:47,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T07:27:47,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T07:27:47,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T07:27:47,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:47,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:47,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:47,955 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T07:27:47,956 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:27:47,956 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T07:27:47,956 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:27:47,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:27:47,956 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:27:47,956 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:27:47,957 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '15ed1af4400d,38495,1732606030549' ***** 2024-11-26T07:27:47,957 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T07:27:47,957 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '15ed1af4400d,42269,1732606030598' ***** 2024-11-26T07:27:47,957 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T07:27:47,957 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T07:27:47,957 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T07:27:47,957 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T07:27:47,957 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T07:27:47,958 INFO [RS:1;15ed1af4400d:38495 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T07:27:47,958 INFO [RS:1;15ed1af4400d:38495 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T07:27:47,958 INFO [RS:2;15ed1af4400d:42269 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T07:27:47,958 INFO [RS:2;15ed1af4400d:42269 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T07:27:47,958 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(3091): Received CLOSE for 28da289971d201b5d537d47956344363 2024-11-26T07:27:47,958 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(959): stopping server 15ed1af4400d,38495,1732606030549 2024-11-26T07:27:47,958 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(959): stopping server 15ed1af4400d,42269,1732606030598 2024-11-26T07:27:47,958 INFO [RS:1;15ed1af4400d:38495 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:27:47,958 INFO [RS:2;15ed1af4400d:42269 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:27:47,958 INFO [RS:1;15ed1af4400d:38495 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;15ed1af4400d:38495. 2024-11-26T07:27:47,958 INFO [RS:2;15ed1af4400d:42269 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;15ed1af4400d:42269. 2024-11-26T07:27:47,958 DEBUG [RS:1;15ed1af4400d:38495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:27:47,958 DEBUG [RS:1;15ed1af4400d:38495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:27:47,958 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 28da289971d201b5d537d47956344363, disabling compactions & flushes 2024-11-26T07:27:47,958 DEBUG [RS:2;15ed1af4400d:42269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:27:47,958 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T07:27:47,958 DEBUG [RS:2;15ed1af4400d:42269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:27:47,958 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T07:27:47,958 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:47,958 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T07:27:47,958 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:47,958 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(976): stopping server 15ed1af4400d,42269,1732606030598; all regions closed. 2024-11-26T07:27:47,958 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. after waiting 0 ms 2024-11-26T07:27:47,958 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:47,958 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T07:27:47,959 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T07:27:47,959 DEBUG [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 28da289971d201b5d537d47956344363=testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363.} 2024-11-26T07:27:47,959 DEBUG [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 28da289971d201b5d537d47956344363 2024-11-26T07:27:47,959 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T07:27:47,959 INFO [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T07:27:47,959 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T07:27:47,959 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T07:27:47,959 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T07:27:47,960 INFO [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-26T07:27:47,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741834_1010 (size=93) 2024-11-26T07:27:47,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741834_1010 (size=93) 2024-11-26T07:27:47,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741834_1010 (size=93) 2024-11-26T07:27:47,966 DEBUG [RS:2;15ed1af4400d:42269 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/oldWALs 2024-11-26T07:27:47,966 INFO [RS:2;15ed1af4400d:42269 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 15ed1af4400d%2C42269%2C1732606030598:(num 1732606031953) 2024-11-26T07:27:47,966 DEBUG [RS:2;15ed1af4400d:42269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:27:47,966 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:27:47,966 INFO [RS:2;15ed1af4400d:42269 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:27:47,967 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/default/testReplayEditsAfterRegionMovedWithMultiCF/28da289971d201b5d537d47956344363/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-26T07:27:47,967 INFO [RS:2;15ed1af4400d:42269 {}] hbase.ChoreService(370): Chore service for: regionserver/15ed1af4400d:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T07:27:47,967 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T07:27:47,967 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T07:27:47,967 INFO [regionserver/15ed1af4400d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:27:47,967 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T07:27:47,967 INFO [RS:2;15ed1af4400d:42269 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:27:47,967 INFO [RS:2;15ed1af4400d:42269 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42269 2024-11-26T07:27:47,968 INFO [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:47,968 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 28da289971d201b5d537d47956344363: Waiting for close lock at 1732606067958Running coprocessor pre-close hooks at 1732606067958Disabling compacts and flushes for region at 1732606067958Disabling writes for close at 1732606067958Writing region close event to WAL at 1732606067961 (+3 ms)Running coprocessor post-close hooks at 1732606067968 (+7 ms)Closed at 1732606067968 2024-11-26T07:27:47,968 DEBUG [RS_CLOSE_REGION-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363. 2024-11-26T07:27:47,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/15ed1af4400d,42269,1732606030598 2024-11-26T07:27:47,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:27:47,969 INFO [RS:2;15ed1af4400d:42269 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:27:47,971 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [15ed1af4400d,42269,1732606030598] 2024-11-26T07:27:47,975 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/15ed1af4400d,42269,1732606030598 already deleted, retry=false 2024-11-26T07:27:47,975 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 15ed1af4400d,42269,1732606030598 expired; onlineServers=1 2024-11-26T07:27:47,985 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/.tmp/info/7de794f1613a4c6b9db7b11a18362cf1 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732606044994.28da289971d201b5d537d47956344363./info:regioninfo/1732606048037/Put/seqid=0 2024-11-26T07:27:47,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741938_1118 (size=8243) 2024-11-26T07:27:47,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741938_1118 (size=8243) 2024-11-26T07:27:47,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741938_1118 (size=8243) 2024-11-26T07:27:47,992 INFO [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/.tmp/info/7de794f1613a4c6b9db7b11a18362cf1 2024-11-26T07:27:48,011 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/.tmp/ns/a283f515313841bf9f78bfcc1d6368e0 is 43, key is default/ns:d/1732606032530/Put/seqid=0 2024-11-26T07:27:48,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741939_1119 (size=5153) 2024-11-26T07:27:48,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741939_1119 (size=5153) 2024-11-26T07:27:48,017 INFO [regionserver/15ed1af4400d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:27:48,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741939_1119 (size=5153) 2024-11-26T07:27:48,018 INFO [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/.tmp/ns/a283f515313841bf9f78bfcc1d6368e0 2024-11-26T07:27:48,037 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/.tmp/table/aa28e16479f9441bb70f39f637a161f7 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732606045411/Put/seqid=0 2024-11-26T07:27:48,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741940_1120 (size=5431) 2024-11-26T07:27:48,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741940_1120 (size=5431) 2024-11-26T07:27:48,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741940_1120 (size=5431) 2024-11-26T07:27:48,045 INFO [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/.tmp/table/aa28e16479f9441bb70f39f637a161f7 2024-11-26T07:27:48,047 INFO [regionserver/15ed1af4400d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:27:48,052 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/.tmp/info/7de794f1613a4c6b9db7b11a18362cf1 as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/info/7de794f1613a4c6b9db7b11a18362cf1 2024-11-26T07:27:48,057 INFO [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/info/7de794f1613a4c6b9db7b11a18362cf1, entries=18, sequenceid=21, filesize=8.0 K 2024-11-26T07:27:48,057 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/.tmp/ns/a283f515313841bf9f78bfcc1d6368e0 as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/ns/a283f515313841bf9f78bfcc1d6368e0 2024-11-26T07:27:48,062 INFO [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/ns/a283f515313841bf9f78bfcc1d6368e0, entries=2, sequenceid=21, filesize=5.0 K 2024-11-26T07:27:48,063 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/.tmp/table/aa28e16479f9441bb70f39f637a161f7 as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/table/aa28e16479f9441bb70f39f637a161f7 2024-11-26T07:27:48,068 INFO [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/table/aa28e16479f9441bb70f39f637a161f7, entries=2, sequenceid=21, filesize=5.3 K 2024-11-26T07:27:48,069 INFO [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=21, compaction requested=false 2024-11-26T07:27:48,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:27:48,071 INFO [RS:2;15ed1af4400d:42269 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:27:48,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42269-0x101532bd7460003, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:27:48,071 INFO [RS:2;15ed1af4400d:42269 {}] regionserver.HRegionServer(1031): Exiting; stopping=15ed1af4400d,42269,1732606030598; zookeeper connection closed. 2024-11-26T07:27:48,071 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47606132 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47606132 2024-11-26T07:27:48,073 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-26T07:27:48,074 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T07:27:48,074 INFO [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T07:27:48,074 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732606067959Running coprocessor pre-close hooks at 1732606067959Disabling compacts and flushes for region at 1732606067959Disabling writes for close at 1732606067959Obtaining lock to block concurrent updates at 1732606067960 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732606067960Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732606067960Flushing stores of hbase:meta,,1.1588230740 at 1732606067963 (+3 ms)Flushing 1588230740/info: creating writer at 1732606067963Flushing 1588230740/info: appending metadata at 1732606067984 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732606067984Flushing 1588230740/ns: creating writer at 1732606067997 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732606068010 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732606068011 (+1 ms)Flushing 1588230740/table: creating writer at 1732606068023 (+12 ms)Flushing 1588230740/table: appending metadata at 1732606068037 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732606068037Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75dc10ef: reopening flushed file at 1732606068051 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73502bbd: reopening flushed file at 1732606068057 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4edac18b: reopening flushed file at 1732606068062 (+5 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=21, compaction requested=false at 1732606068069 (+7 ms)Writing region close event to WAL at 1732606068070 (+1 ms)Running coprocessor post-close hooks at 1732606068073 (+3 ms)Closed at 1732606068074 (+1 ms) 2024-11-26T07:27:48,074 DEBUG [RS_CLOSE_META-regionserver/15ed1af4400d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T07:27:48,159 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(976): stopping server 15ed1af4400d,38495,1732606030549; all regions closed. 2024-11-26T07:27:48,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741836_1012 (size=3783) 2024-11-26T07:27:48,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741836_1012 (size=3783) 2024-11-26T07:27:48,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741836_1012 (size=3783) 2024-11-26T07:27:48,164 DEBUG [RS:1;15ed1af4400d:38495 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/oldWALs 2024-11-26T07:27:48,164 INFO [RS:1;15ed1af4400d:38495 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 15ed1af4400d%2C38495%2C1732606030549.meta:.meta(num 1732606032376) 2024-11-26T07:27:48,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741835_1011 (size=2652) 2024-11-26T07:27:48,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741835_1011 (size=2652) 2024-11-26T07:27:48,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741835_1011 (size=2652) 2024-11-26T07:27:48,169 DEBUG [RS:1;15ed1af4400d:38495 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/oldWALs 2024-11-26T07:27:48,169 INFO [RS:1;15ed1af4400d:38495 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 15ed1af4400d%2C38495%2C1732606030549:(num 1732606031953) 2024-11-26T07:27:48,169 DEBUG [RS:1;15ed1af4400d:38495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:27:48,169 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:27:48,169 INFO [RS:1;15ed1af4400d:38495 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:27:48,169 INFO [RS:1;15ed1af4400d:38495 {}] hbase.ChoreService(370): Chore service for: regionserver/15ed1af4400d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T07:27:48,169 INFO [RS:1;15ed1af4400d:38495 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:27:48,169 INFO [regionserver/15ed1af4400d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:27:48,170 INFO [RS:1;15ed1af4400d:38495 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38495 2024-11-26T07:27:48,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/15ed1af4400d,38495,1732606030549 2024-11-26T07:27:48,172 INFO [RS:1;15ed1af4400d:38495 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:27:48,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:27:48,173 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [15ed1af4400d,38495,1732606030549] 2024-11-26T07:27:48,174 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/15ed1af4400d,38495,1732606030549 already deleted, retry=false 2024-11-26T07:27:48,174 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 15ed1af4400d,38495,1732606030549 expired; onlineServers=0 2024-11-26T07:27:48,174 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '15ed1af4400d,45945,1732606029786' ***** 2024-11-26T07:27:48,174 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T07:27:48,174 INFO [M:0;15ed1af4400d:45945 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:27:48,174 INFO [M:0;15ed1af4400d:45945 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:27:48,175 DEBUG [M:0;15ed1af4400d:45945 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T07:27:48,175 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T07:27:48,175 DEBUG [M:0;15ed1af4400d:45945 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T07:27:48,175 DEBUG [master/15ed1af4400d:0:becomeActiveMaster-HFileCleaner.large.0-1732606031600 {}] cleaner.HFileCleaner(306): Exit Thread[master/15ed1af4400d:0:becomeActiveMaster-HFileCleaner.large.0-1732606031600,5,FailOnTimeoutGroup] 2024-11-26T07:27:48,175 DEBUG [master/15ed1af4400d:0:becomeActiveMaster-HFileCleaner.small.0-1732606031614 {}] cleaner.HFileCleaner(306): Exit Thread[master/15ed1af4400d:0:becomeActiveMaster-HFileCleaner.small.0-1732606031614,5,FailOnTimeoutGroup] 2024-11-26T07:27:48,175 INFO [M:0;15ed1af4400d:45945 {}] hbase.ChoreService(370): Chore service for: master/15ed1af4400d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T07:27:48,175 INFO [M:0;15ed1af4400d:45945 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:27:48,175 DEBUG [M:0;15ed1af4400d:45945 {}] master.HMaster(1795): Stopping service threads 2024-11-26T07:27:48,175 INFO [M:0;15ed1af4400d:45945 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T07:27:48,175 INFO [M:0;15ed1af4400d:45945 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T07:27:48,176 INFO [M:0;15ed1af4400d:45945 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T07:27:48,176 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T07:27:48,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T07:27:48,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:27:48,178 DEBUG [M:0;15ed1af4400d:45945 {}] zookeeper.ZKUtil(347): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T07:27:48,178 WARN [M:0;15ed1af4400d:45945 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T07:27:48,178 INFO [M:0;15ed1af4400d:45945 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/.lastflushedseqids 2024-11-26T07:27:48,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741941_1121 (size=138) 2024-11-26T07:27:48,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741941_1121 (size=138) 2024-11-26T07:27:48,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741941_1121 (size=138) 2024-11-26T07:27:48,191 INFO [M:0;15ed1af4400d:45945 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T07:27:48,191 INFO [M:0;15ed1af4400d:45945 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T07:27:48,191 DEBUG [M:0;15ed1af4400d:45945 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T07:27:48,191 INFO [M:0;15ed1af4400d:45945 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:27:48,191 DEBUG [M:0;15ed1af4400d:45945 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:27:48,192 DEBUG [M:0;15ed1af4400d:45945 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T07:27:48,192 DEBUG [M:0;15ed1af4400d:45945 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:27:48,192 INFO [M:0;15ed1af4400d:45945 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.33 KB heapSize=83.72 KB 2024-11-26T07:27:48,207 DEBUG [M:0;15ed1af4400d:45945 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/974c624f0e8d4c36be1135c2c67d5a25 is 82, key is hbase:meta,,1/info:regioninfo/1732606032457/Put/seqid=0 2024-11-26T07:27:48,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741942_1122 (size=5672) 2024-11-26T07:27:48,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741942_1122 (size=5672) 2024-11-26T07:27:48,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741942_1122 (size=5672) 2024-11-26T07:27:48,213 INFO [M:0;15ed1af4400d:45945 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/974c624f0e8d4c36be1135c2c67d5a25 2024-11-26T07:27:48,234 DEBUG [M:0;15ed1af4400d:45945 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/975b4c3a150d469a868ab3e129fb2615 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732606045416/Put/seqid=0 2024-11-26T07:27:48,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741943_1123 (size=7754) 2024-11-26T07:27:48,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741943_1123 (size=7754) 2024-11-26T07:27:48,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741943_1123 (size=7754) 2024-11-26T07:27:48,249 INFO [M:0;15ed1af4400d:45945 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/975b4c3a150d469a868ab3e129fb2615 2024-11-26T07:27:48,253 INFO [M:0;15ed1af4400d:45945 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 975b4c3a150d469a868ab3e129fb2615 2024-11-26T07:27:48,268 DEBUG [M:0;15ed1af4400d:45945 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2bd8b8f5691e40fdb3bfed394359988d is 69, key is 15ed1af4400d,38495,1732606030549/rs:state/1732606031698/Put/seqid=0 2024-11-26T07:27:48,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:27:48,273 INFO [RS:1;15ed1af4400d:38495 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:27:48,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38495-0x101532bd7460002, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:27:48,273 INFO [RS:1;15ed1af4400d:38495 {}] regionserver.HRegionServer(1031): Exiting; stopping=15ed1af4400d,38495,1732606030549; zookeeper connection closed. 2024-11-26T07:27:48,276 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@46e4ffcc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@46e4ffcc 2024-11-26T07:27:48,277 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T07:27:48,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741944_1124 (size=5440) 2024-11-26T07:27:48,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741944_1124 (size=5440) 2024-11-26T07:27:48,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741944_1124 (size=5440) 2024-11-26T07:27:48,280 INFO [M:0;15ed1af4400d:45945 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2bd8b8f5691e40fdb3bfed394359988d 2024-11-26T07:27:48,285 INFO [M:0;15ed1af4400d:45945 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2bd8b8f5691e40fdb3bfed394359988d 2024-11-26T07:27:48,285 DEBUG [M:0;15ed1af4400d:45945 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/974c624f0e8d4c36be1135c2c67d5a25 as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/974c624f0e8d4c36be1135c2c67d5a25 2024-11-26T07:27:48,290 INFO [M:0;15ed1af4400d:45945 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/974c624f0e8d4c36be1135c2c67d5a25, entries=8, sequenceid=168, filesize=5.5 K 2024-11-26T07:27:48,290 DEBUG [M:0;15ed1af4400d:45945 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/975b4c3a150d469a868ab3e129fb2615 as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/975b4c3a150d469a868ab3e129fb2615 2024-11-26T07:27:48,295 INFO [M:0;15ed1af4400d:45945 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 975b4c3a150d469a868ab3e129fb2615 2024-11-26T07:27:48,295 INFO [M:0;15ed1af4400d:45945 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/975b4c3a150d469a868ab3e129fb2615, entries=17, sequenceid=168, filesize=7.6 K 2024-11-26T07:27:48,296 DEBUG [M:0;15ed1af4400d:45945 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2bd8b8f5691e40fdb3bfed394359988d as hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2bd8b8f5691e40fdb3bfed394359988d 2024-11-26T07:27:48,300 INFO [M:0;15ed1af4400d:45945 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2bd8b8f5691e40fdb3bfed394359988d 2024-11-26T07:27:48,300 INFO [M:0;15ed1af4400d:45945 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41363/user/jenkins/test-data/3db8846b-e476-bee7-838a-7d3d01680d42/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2bd8b8f5691e40fdb3bfed394359988d, entries=3, sequenceid=168, filesize=5.3 K 2024-11-26T07:27:48,301 INFO [M:0;15ed1af4400d:45945 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=168, compaction requested=false 2024-11-26T07:27:48,303 INFO [M:0;15ed1af4400d:45945 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:27:48,303 DEBUG [M:0;15ed1af4400d:45945 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732606068191Disabling compacts and flushes for region at 1732606068191Disabling writes for close at 1732606068192 (+1 ms)Obtaining lock to block concurrent updates at 1732606068192Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732606068192Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69972, getHeapSize=85664, getOffHeapSize=0, getCellsCount=195 at 1732606068192Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732606068193 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732606068193Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732606068207 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732606068207Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732606068218 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732606068233 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732606068233Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732606068253 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732606068267 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732606068267Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@664b7ea3: reopening flushed file at 1732606068285 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20689527: reopening flushed file at 1732606068290 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3479ebd8: reopening flushed file at 1732606068295 (+5 ms)Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=168, compaction requested=false at 1732606068301 (+6 ms)Writing region close event to WAL at 1732606068303 (+2 ms)Closed at 1732606068303 2024-11-26T07:27:48,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40873 is added to blk_1073741830_1006 (size=71325) 2024-11-26T07:27:48,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41787 is added to blk_1073741830_1006 (size=71325) 2024-11-26T07:27:48,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42007 is added to blk_1073741830_1006 (size=71325) 2024-11-26T07:27:48,306 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:27:48,306 INFO [M:0;15ed1af4400d:45945 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T07:27:48,306 INFO [M:0;15ed1af4400d:45945 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45945 2024-11-26T07:27:48,307 INFO [M:0;15ed1af4400d:45945 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:27:48,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:27:48,409 INFO [M:0;15ed1af4400d:45945 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:27:48,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45945-0x101532bd7460000, quorum=127.0.0.1:57836, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:27:48,415 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048752 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048752 (inode 16655) Holder DFSClient_NONMAPREDUCE_802469026_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732606048435/wal.1732606048752 (inode 16655) Holder DFSClient_NONMAPREDUCE_802469026_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T07:27:48,416 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732606040567/wal.1732606040669 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:48,417 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732606063309/wal.1732606063535 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:48,419 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606058123 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606058123 (inode 16678) Holder DFSClient_NONMAPREDUCE_802469026_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732606048886/wal.1732606058123 (inode 16678) Holder DFSClient_NONMAPREDUCE_802469026_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T07:27:48,419 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732606033567/wal.1732606033632 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:48,421 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058741 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058741 (inode 16704) Holder DFSClient_NONMAPREDUCE_802469026_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732606058317/wal.1732606058741 (inode 16704) Holder DFSClient_NONMAPREDUCE_802469026_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T07:27:48,422 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732606058892/wal.1732606058949 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:27:48,424 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal.1732606033435 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal.1732606033435 (inode 16485) Holder DFSClient_NONMAPREDUCE_802469026_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732606033258/wal.1732606033435 (inode 16485) Holder DFSClient_NONMAPREDUCE_802469026_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T07:27:48,426 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal.1732606032979 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal.1732606032979 (inode 16462) Holder DFSClient_NONMAPREDUCE_802469026_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732606032732/wal.1732606032979 (inode 16462) Holder DFSClient_NONMAPREDUCE_802469026_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T07:27:48,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@31a39bd8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T07:27:48,455 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5aa63b11{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T07:27:48,455 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T07:27:48,455 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de3b13{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T07:27:48,455 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d458b2a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/hadoop.log.dir/,STOPPED} 2024-11-26T07:27:48,458 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T07:27:48,458 WARN [BP-568013986-172.17.0.2-1732606026764 heartbeating to localhost/127.0.0.1:41363 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T07:27:48,458 WARN [BP-568013986-172.17.0.2-1732606026764 heartbeating to localhost/127.0.0.1:41363 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-568013986-172.17.0.2-1732606026764 (Datanode Uuid cef9386c-28c6-4e41-8f8f-519a1f9c1555) service to localhost/127.0.0.1:41363 2024-11-26T07:27:48,458 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T07:27:48,459 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data5/current/BP-568013986-172.17.0.2-1732606026764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T07:27:48,460 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data6/current/BP-568013986-172.17.0.2-1732606026764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T07:27:48,460 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T07:27:48,463 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@41fe4ca9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T07:27:48,463 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@74ee88d2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T07:27:48,463 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T07:27:48,463 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@424d969a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T07:27:48,463 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@157ee20f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/hadoop.log.dir/,STOPPED} 2024-11-26T07:27:48,464 WARN [BP-568013986-172.17.0.2-1732606026764 heartbeating to localhost/127.0.0.1:41363 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T07:27:48,464 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T07:27:48,464 WARN [BP-568013986-172.17.0.2-1732606026764 heartbeating to localhost/127.0.0.1:41363 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-568013986-172.17.0.2-1732606026764 (Datanode Uuid b95d11b6-f71c-42e0-8cb2-27719df50a40) service to localhost/127.0.0.1:41363 2024-11-26T07:27:48,464 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T07:27:48,465 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data3/current/BP-568013986-172.17.0.2-1732606026764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T07:27:48,465 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data4/current/BP-568013986-172.17.0.2-1732606026764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T07:27:48,465 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T07:27:48,467 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3a1c1498{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T07:27:48,467 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3badc187{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T07:27:48,467 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T07:27:48,467 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@991988e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T07:27:48,468 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@481eb1b6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/hadoop.log.dir/,STOPPED} 2024-11-26T07:27:48,469 WARN [BP-568013986-172.17.0.2-1732606026764 heartbeating to localhost/127.0.0.1:41363 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T07:27:48,469 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T07:27:48,469 WARN [BP-568013986-172.17.0.2-1732606026764 heartbeating to localhost/127.0.0.1:41363 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-568013986-172.17.0.2-1732606026764 (Datanode Uuid 62e50c5b-5dcc-4a7a-a637-006363e140ae) service to localhost/127.0.0.1:41363 2024-11-26T07:27:48,469 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T07:27:48,469 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data1/current/BP-568013986-172.17.0.2-1732606026764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T07:27:48,470 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/cluster_324997db-0ef0-1501-4f88-7757fe4a9a01/data/data2/current/BP-568013986-172.17.0.2-1732606026764 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T07:27:48,470 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T07:27:48,480 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@12264dc8{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T07:27:48,480 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ce294cb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T07:27:48,481 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T07:27:48,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43ac9415{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T07:27:48,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@77da9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e36cbbe6-5cc7-cd2e-5d8b-70a932b26a78/hadoop.log.dir/,STOPPED} 2024-11-26T07:27:48,491 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T07:27:48,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down